refactor(workers): unify cloud bootstrap lifecycle (#129719)

* refactor(workers): unify cloud bootstrap lifecycle

* fix(workers): preserve cloud setup and repair qa fixtures
This commit is contained in:
Peter Steinberger
2026-08-25 20:11:33 -07:00
committed by GitHub
parent b3d3494919
commit 422d6818e4
11 changed files with 107 additions and 123 deletions
+12 -1
View File
@@ -138,6 +138,17 @@ OpenClaw derives one canonical `cbx_...` lease ID from the durable provision ope
`settings.setup` runs on the leased box after Crabbox reports it ready and before ephemeral node enrollment. It runs on **every** provision attempt, including replay after an interrupted dispatch, so it must be idempotent — guard installs with a `command -v`/`test -x` check as in the example. At minimum, the resulting machine needs Node.js and `npx`. If setup or enrollment fails, the provider stops the lease and the dispatch fails closed; no half-configured paid box is hidden behind terminal state.
The example profile above prepares OpenClaw worker turns only. To make the same profile Codex-ready, run the following on the Gateway host. It preserves the existing setup, captures the Gateway version, installs that exact OpenClaw release on each cloud node, and installs the matching Codex plugin through the trusted official npm path:
```bash
gateway_version="$(openclaw --version | awk '{print $2}')"
existing_setup="$(openclaw config get cloudWorkers.profiles.aws.settings.setup)"
openclaw config set cloudWorkers.profiles.aws.settings.setup \
"${existing_setup}; sudo npm install -g openclaw@${gateway_version}; openclaw plugins install npm:@openclaw/codex@${gateway_version} --pin --force"
```
`--force` allows setup replay to converge when the plugin is already installed. After upgrading the Gateway, update both appended package versions to match it while preserving the rest of the setup. Unreleased source builds require exact locally packed packages instead of registry versions.
### Bundle installation
The setup bootstrap first reuses an installed `openclaw` binary when its version exactly matches the Gateway, then tries the exact `openclaw@<version>` registry package. For an unreleased source build, install a locally packed candidate with the same version in `settings.setup`; the provider will select it before touching the registry. After the node connects and publishes its session-host inventory, the Gateway pushes one content-addressed worker bundle through the paired channel. The node verifies and publishes those exact bytes without installing the normal OpenClaw package dependency tree. A stale Gateway build retires the environment and reprovisions against the current bundle rather than downgrading the execution-context protocol.
@@ -355,7 +366,7 @@ The desktop never gains public ingress. The node reads `/var/lib/crabbox/vnc.pas
- **No cloud profile is advertised** — run the `operator.read`-scoped `openclaw gateway call environments.list --params '{}'`. If the response has no `profiles`, ask an administrator to validate `cloudWorkers.profiles`, inspect the provider plugin, and restart the Gateway. This is a configuration or provider-activation problem, not an authorization result.
- **Cloud destinations are hidden or an RPC is denied** — cloud profile dispatch and profile-target moves require `operator.admin`. `operator.write` can dispatch or move to an eligible paired device, move to the Gateway, and reclaim a placement; `operator.read` alone can discover profiles but cannot start, stop, or move a session. Profile configuration, infrastructure pairing, Connect machine, raw environment lifecycle, direct `execNode` execution, incognito sessions, and arbitrary host or node paths remain `operator.admin`.
- **The selected runtime lacks cloud placement support** — choose a model whose advertised runtime supports cloud placement. The bundled OpenClaw and Codex runtimes are supported; undeclared runtimes remain local-only.
- **Codex cannot use a cloud profile** — verify that the profile advertises `remote-exec`, the Gateway enables the Codex plugin, and `gateway.nodes.commands.allow` includes `codex.exec-server.stdio.v1` without a matching deny rule. Approve the exact node invocation when prompted. Codex does not require an available OpenClaw worker slot; a missing or denied command must be corrected rather than bypassed with Gateway or SSH execution.
- **Codex cannot use a cloud profile** — verify that the profile advertises `remote-exec`, its setup installs the exact official `npm:@openclaw/codex@<gateway-version>` plugin, the Gateway enables the Codex plugin, and `gateway.nodes.commands.allow` includes `codex.exec-server.stdio.v1` without a matching deny rule. Approve the exact node invocation when prompted. Codex does not require an available OpenClaw worker slot; a missing plugin or denied command must be corrected rather than bypassed with Gateway or SSH execution.
- **"Worker bootstrap requires Node.js on the leased host"** — add a Node install to `settings.setup` (see above).
- **AWS instance-role attestation fails** — clear `aws.instanceProfile` (and `CRABBOX_AWS_INSTANCE_PROFILE`, if set). Install Crabbox 0.41.1 or newer; older binaries do not satisfy the fixed-ID and authoritative `providerMetadata.instanceProfileAttached` contracts required for AWS admission.
- **Dispatch or workspace recovery fails** — inspect `environments.list` and `sessions.describe`. A failed environment exposes its bounded environment error. A failed placement exposes `recoveryError` plus its durable per-session `terminalReason`; the selected Control UI chat shows that terminal reason above the composer. When deeper diagnosis is necessary, an operator on the Gateway host can inspect the durable worker state read-only. Do not edit the state database to bypass lifecycle fencing.
@@ -38,7 +38,7 @@ export function createCrabboxNodeEnrollmentSetup(params: {
enrollment.mode === "connect"
? `connect --target-file "$setup_code_file" --ephemeral --display-name ${shellQuote(enrollment.displayName)}`
: `node run --ephemeral --display-name ${shellQuote(enrollment.displayName)}`;
const prepareCodex = (binary: string): string[] => {
const prepareCodex = (): string[] => {
if (executionMode !== "remote-exec") {
return [];
}
@@ -71,8 +71,8 @@ export function createCrabboxNodeEnrollmentSetup(params: {
"}",
].join("");
return [
` ${binary} plugins inspect codex --json | node -e ${shellQuote(inspectPlugin)} "$state_dir"`,
` OPENCLAW_STATE_DIR="$state_dir" ${binary} plugins enable codex`,
`"$@" plugins inspect codex --json | node -e ${shellQuote(inspectPlugin)} "$state_dir"`,
'OPENCLAW_STATE_DIR="$state_dir" "$@" plugins enable codex',
];
};
const command = [
@@ -106,12 +106,12 @@ export function createCrabboxNodeEnrollmentSetup(params: {
"fi",
'package_spec="$(cat "$package_spec_file")"',
'if [ "$package_spec" = "@global" ]; then',
...prepareCodex("openclaw"),
` setsid -f sh -c 'printf "%s\\n" "$$" >"$1"; shift; exec "$@"' sh "$pid_file" env OPENCLAW_STATE_DIR="$state_dir" openclaw ${launch} >"$state_dir/node.log" 2>&1 </dev/null`,
" set -- openclaw",
"else",
...prepareCodex('npx --yes --package "$package_spec" -- openclaw'),
` setsid -f sh -c 'printf "%s\\n" "$$" >"$1"; shift; exec "$@"' sh "$pid_file" env OPENCLAW_STATE_DIR="$state_dir" npx --yes --package "$package_spec" -- openclaw ${launch} >"$state_dir/node.log" 2>&1 </dev/null`,
' set -- npx --yes --package "$package_spec" -- openclaw',
"fi",
...prepareCodex(),
`setsid -f sh -c 'printf "%s\\n" "$$" >"$1"; shift; exec "$@"' sh "$pid_file" env OPENCLAW_STATE_DIR="$state_dir" "$@" ${launch} >"$state_dir/node.log" 2>&1 </dev/null`,
'for _ in 1 2 3 4 5 6 7 8 9 10; do [ -s "$pid_file" ] && break; sleep 0.1; done',
'test -s "$pid_file"',
].join("\n");
@@ -429,10 +429,11 @@ describe("Crabbox worker provider", () => {
expect(setup).toContain('[launcher,"--version"]');
expect(setup).toContain("codex-cli ${runtime.version}");
expect(setup).not.toContain("$state_dir/extensions/codex");
expect(setup).toContain('OPENCLAW_STATE_DIR="$state_dir" openclaw plugins enable codex');
expect(setup).toContain(
'OPENCLAW_STATE_DIR="$state_dir" npx --yes --package "$package_spec" -- openclaw plugins enable codex',
);
expect(setup).toContain("set -- openclaw");
expect(setup).toContain('set -- npx --yes --package "$package_spec" -- openclaw');
expect(setup).toContain('OPENCLAW_STATE_DIR="$state_dir" "$@" plugins enable codex');
expect(setup.match(/plugins inspect codex --json/g)).toHaveLength(1);
expect(setup.match(/setsid -f sh -c/g)).toHaveLength(1);
expect(setup.indexOf("plugins inspect codex --json")).toBeGreaterThan(
setup.indexOf(setupCodeCleared),
);
@@ -15,10 +15,12 @@ describe("worker environment service", () => {
support.setupWorkerEnvironmentServiceSuite();
it("fails node provisioning visibly when Gateway bundle installation fails", async () => {
const destroy = vi.fn(async () => {});
const workerService = support.createService(
support.createProvider({
supportedExecutionModes: ["worker-turn"],
provisionBeforeInstallation: true,
destroy,
provision: async () => ({
leaseId: "device-lease-install-failure",
node: { deviceId: "device-1" },
@@ -35,10 +37,62 @@ describe("worker environment service", () => {
workerService.create("development", "request-device-install-failure"),
).rejects.toMatchObject({
code: "bootstrap_failure",
message: expect.stringContaining("bundle transfer unavailable"),
message: "Worker node bootstrap failed: bundle transfer unavailable",
} satisfies Partial<WorkerEnvironmentServiceError>);
expect(destroy).toHaveBeenCalledWith({
leaseId: "device-lease-install-failure",
profile: { region: "test" },
});
expect(support.testState.store.list()[0]).toMatchObject({
state: "failed",
leaseId: null,
nodeDeviceId: null,
lastError: expect.stringContaining("bundle transfer unavailable"),
});
});
it("keeps an indeterminate node bootstrap teardown retryable", async () => {
let teardownFails = true;
const workerService = support.createService(
support.createProvider({
supportedExecutionModes: ["worker-turn"],
provisionBeforeInstallation: true,
provision: async () => ({
leaseId: "device-lease-cleanup-failure",
node: { deviceId: "device-cleanup-failure" },
}),
destroy: async () => {
if (teardownFails) {
throw new Error("provider teardown timed out");
}
},
}),
{
ensureNodeWorkerBundle: async () => {
throw new Error("bundle transfer unavailable");
},
},
);
await expect(
workerService.create("development", "request-device-cleanup"),
).rejects.toMatchObject({
code: "bootstrap_failure",
message: "Worker node bootstrap failed; teardown is pending: bundle transfer unavailable",
} satisfies Partial<WorkerEnvironmentServiceError>);
expect(support.testState.store.list()[0]).toMatchObject({
state: "destroying",
leaseId: "device-lease-cleanup-failure",
nodeDeviceId: "device-cleanup-failure",
teardownTerminalState: "failed",
});
teardownFails = false;
await workerService.reconcileOnce();
expect(support.testState.store.list()[0]).toMatchObject({
state: "failed",
leaseId: null,
nodeDeviceId: null,
lastError: expect.stringContaining("bundle transfer unavailable"),
});
});
@@ -136,7 +136,9 @@ export function createWorkerProviderLifecycle(options: WorkerProviderLifecycleOp
const failureLabel =
failureCode === "invalid_profile"
? "Worker provider returned an incompatible lease"
: "Worker bootstrap failed";
: leasePatch?.nodeDeviceId
? "Worker node bootstrap failed"
: "Worker bootstrap failed";
const requested = store.requestDestroy({
environmentId: record.environmentId,
state: record.state,
@@ -184,18 +186,10 @@ export function createWorkerProviderLifecycle(options: WorkerProviderLifecycleOp
};
const finishNodeProvisioning = createWorkerNodeProvisioning({
store,
tunnels,
ensureNodeWorkerBundle: options.ensureNodeWorkerBundle,
commitReady,
move,
destroyProviderLease: async (record, leaseId, provider) =>
await callProvider(record.environmentId, () =>
provider.destroy(lifecycleLease(record, leaseId)),
),
finishProvenDestroy,
saveError,
serviceError,
failBootstrap: async (record, leaseId, provider, error, patch) =>
await failBootstrap(record, leaseId, provider, error, "bootstrap_failure", patch),
});
const finishBootstrap = async (
@@ -1,78 +1,34 @@
import type { WorkerAdmissionHandshake } from "../../../packages/gateway-protocol/src/schema/worker-admission.js";
import type { WorkerLease, WorkerProvider } from "../../plugins/types.js";
import type { WorkerCredentialBroker } from "./credential-broker.js";
import type { WorkerEnvironmentRecord, WorkerEnvironmentStore } from "./store.js";
import type { WorkerTunnelManager } from "./tunnel.js";
import { boundedWorkerError as boundedError } from "./worker-error.js";
import type { WorkerEnvironmentRecord, WorkerEnvironmentTransitionPatch } from "./store.js";
type NodeLease = Extract<WorkerLease, { node: { deviceId: string } }>;
type WorkerNodeProvisioningOptions = {
store: WorkerEnvironmentStore;
tunnels?: Pick<WorkerTunnelManager, "stop">;
ensureNodeWorkerBundle?: (deviceId: string) => Promise<WorkerAdmissionHandshake>;
commitReady: WorkerCredentialBroker["commitReady"];
move: (
record: WorkerEnvironmentRecord,
to: "draining" | "destroying",
patch?: Parameters<WorkerEnvironmentStore["transition"]>[0]["patch"],
) => WorkerEnvironmentRecord;
destroyProviderLease: (
failBootstrap: (
record: WorkerEnvironmentRecord,
leaseId: string,
provider: WorkerProvider,
) => Promise<void>;
finishProvenDestroy: (record: WorkerEnvironmentRecord) => Promise<WorkerEnvironmentRecord>;
saveError: (record: WorkerEnvironmentRecord, error: unknown) => WorkerEnvironmentRecord;
serviceError: (code: "bootstrap_failure", message: string) => Error;
error: unknown,
patch: WorkerEnvironmentTransitionPatch,
) => Promise<never>;
};
export function createWorkerNodeProvisioning(options: WorkerNodeProvisioningOptions) {
const fail = async (
record: WorkerEnvironmentRecord,
lease: NodeLease,
provider: WorkerProvider,
error: unknown,
): Promise<never> => {
const detail = boundedError(error);
const requested = options.store.requestDestroy({
environmentId: record.environmentId,
state: record.state,
terminalState: "failed",
lastError: detail,
});
const draining = options.move(requested, "draining", {
leaseId: lease.leaseId,
nodeDeviceId: lease.node.deviceId,
sshEndpoint: null,
sharedHost: lease.sharedHost === true,
desktop: lease.desktop ?? null,
lastError: detail,
});
await options.tunnels?.stop(record.environmentId);
const destroying = options.move(draining, "destroying", { lastError: detail });
try {
await options.destroyProviderLease(record, lease.leaseId, provider);
} catch (cleanupError: unknown) {
options.saveError(
destroying,
new Error(`${detail}; provider teardown pending: ${boundedError(cleanupError)}`),
);
throw options.serviceError(
"bootstrap_failure",
`Worker node bootstrap failed; teardown is pending: ${detail}`,
);
}
await options.finishProvenDestroy(destroying);
throw options.serviceError("bootstrap_failure", `Worker node bootstrap failed: ${detail}`);
};
return async (
record: WorkerEnvironmentRecord,
lease: NodeLease,
provider: WorkerProvider,
patch: { leaseId: string; sharedHost: boolean; desktop: WorkerLease["desktop"] | null },
): Promise<WorkerEnvironmentRecord> => {
const nodePatch = {
...patch,
nodeDeviceId: lease.node.deviceId,
sshEndpoint: null,
};
let nodeBuild: WorkerAdmissionHandshake;
try {
if (!options.ensureNodeWorkerBundle) {
@@ -80,16 +36,8 @@ export function createWorkerNodeProvisioning(options: WorkerNodeProvisioningOpti
}
nodeBuild = await options.ensureNodeWorkerBundle(lease.node.deviceId);
} catch (error) {
return await fail(record, lease, provider, error);
return await options.failBootstrap(record, lease.leaseId, provider, error, nodePatch);
}
return options.commitReady(
record,
{ ...nodeBuild, installKind: "bundle" },
{
...patch,
nodeDeviceId: lease.node.deviceId,
sshEndpoint: null,
},
);
return options.commitReady(record, { ...nodeBuild, installKind: "bundle" }, nodePatch);
};
}
+4 -4
View File
@@ -520,8 +520,8 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService
}
return environmentAccess.project(
await providerLifecycle.createWithProfile(profileId, idempotencyKey, {
...(machineClass === undefined ? {} : { machineClass }),
...(executionMode === undefined ? {} : { executionMode }),
machineClass,
executionMode,
}),
);
},
@@ -538,8 +538,8 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService
providerId: profile.providerId,
profileSnapshot: profile.profileSnapshot,
},
...(machineClass === undefined ? {} : { machineClass }),
...(executionMode === undefined ? {} : { executionMode }),
machineClass,
executionMode,
}),
);
},
+4 -11
View File
@@ -23,12 +23,10 @@ describe("draftCloudProfileSupportsExecutionMode", () => {
);
it("does not treat the singular display projection as a placement capability", () => {
expect(
draftCloudProfileSupportsExecutionMode(
{ id: "legacy", providerId: "crabbox", executionMode: "worker-turn" },
"worker-turn",
),
).toBe(false);
const [profile] = readDraftCloudProfiles([
{ id: "legacy", providerId: "crabbox", executionMode: "worker-turn" },
]);
expect(draftCloudProfileSupportsExecutionMode(profile!, "worker-turn")).toBe(false);
});
});
@@ -79,7 +77,6 @@ describe("readDraftCloudProfiles", () => {
id: "aws",
providerId: "crabbox",
trust: "persistent",
executionMode: "worker-turn",
executionModes: ["worker-turn", "remote-exec"],
machines: [
{
@@ -96,19 +93,16 @@ describe("readDraftCloudProfiles", () => {
id: "invalid-trust",
providerId: "crabbox",
trust: undefined,
executionMode: undefined,
},
{
id: "legacy",
providerId: "static-ssh",
trust: undefined,
executionMode: undefined,
},
{
id: "zeta",
providerId: "static-ssh",
trust: "disposable",
executionMode: "worker-turn",
},
]);
});
@@ -136,7 +130,6 @@ describe("readDraftCloudProfiles", () => {
id: "aws",
providerId: "crabbox",
trust: undefined,
executionMode: "remote-exec",
executionModes: [],
},
]);
-7
View File
@@ -29,7 +29,6 @@ export type DraftCloudProfile = {
id: string;
providerId: string;
trust?: "persistent" | "disposable";
executionMode?: WorkerExecutionMode;
executionModes?: readonly WorkerExecutionMode[];
machines?: DraftMachineOption[];
};
@@ -118,7 +117,6 @@ export function readDraftCloudProfiles(value: unknown): DraftCloudProfile[] {
id?: unknown;
providerId?: unknown;
trust?: unknown;
executionMode?: unknown;
executionModes?: unknown;
machines?: unknown;
};
@@ -131,17 +129,12 @@ export function readDraftCloudProfiles(value: unknown): DraftCloudProfile[] {
profile.trust === "persistent" || profile.trust === "disposable"
? profile.trust
: undefined;
const executionMode: WorkerExecutionMode | undefined =
profile.executionMode === "worker-turn" || profile.executionMode === "remote-exec"
? profile.executionMode
: undefined;
const machines = readDraftMachineOptions(profile.machines);
return [
{
id,
providerId,
trust,
executionMode,
...(Object.hasOwn(profile, "executionModes")
? { executionModes: readDraftCloudProfileExecutionModes(profile.executionModes) }
: {}),
@@ -122,7 +122,6 @@ describe("DraftPlaceState cloud machine selection", () => {
{
id: "aws",
providerId: "crabbox",
executionMode: "worker-turn",
executionModes,
machines: [
{ id: "standard", label: "Standard", default: true },
@@ -14,7 +14,6 @@ describe("new-session model runtime placement", () => {
cloudPlacementExecutionMode: "remote-exec" as const,
source: "model" as const,
},
executionMode: "worker-turn" as const,
executionModes: ["worker-turn"] as const,
expected:
"The codex runtime cannot use this cloud worker. Choose a compatible cloud worker or run locally.",
@@ -27,7 +26,6 @@ describe("new-session model runtime placement", () => {
cloudPlacementExecutionMode: "worker-turn" as const,
source: "model" as const,
},
executionMode: "worker-turn" as const,
executionModes: ["worker-turn"] as const,
expected: undefined,
},
@@ -39,19 +37,17 @@ describe("new-session model runtime placement", () => {
cloudPlacementExecutionMode: "worker-turn" as const,
source: "model" as const,
},
executionMode: "worker-turn" as const,
executionModes: ["worker-turn", "remote-exec"] as const,
expected: undefined,
},
{
name: "accepts a remote-exec runtime on the same worker-turn-primary profile",
name: "accepts a remote-exec runtime on the same dual-mode profile",
runtime: {
id: "codex",
cloudPlacementSupported: true,
cloudPlacementExecutionMode: "remote-exec" as const,
source: "model" as const,
},
executionMode: "worker-turn" as const,
executionModes: ["worker-turn", "remote-exec"] as const,
expected: undefined,
},
@@ -63,20 +59,18 @@ describe("new-session model runtime placement", () => {
cloudPlacementExecutionMode: "remote-exec" as const,
source: "model" as const,
},
executionMode: "worker-turn" as const,
executionModes: ["worker-turn"] as const,
expected:
"The codex runtime cannot use this cloud worker. Choose a compatible cloud worker or run locally.",
},
{
name: "does not fall back to the primary mode when an explicit mode set is empty",
name: "rejects an empty placement mode set",
runtime: {
id: "codex",
cloudPlacementSupported: true,
cloudPlacementExecutionMode: "remote-exec" as const,
source: "model" as const,
},
executionMode: "remote-exec" as const,
executionModes: [] as const,
expected:
"The codex runtime cannot use this cloud worker. Choose a compatible cloud worker or run locally.",
@@ -89,21 +83,18 @@ describe("new-session model runtime placement", () => {
cloudPlacementExecutionMode: "remote-exec" as const,
source: "model" as const,
},
executionMode: undefined,
expected:
"The codex runtime cannot use this cloud worker. Choose a compatible cloud worker or run locally.",
},
{
name: "retains the existing whole-runtime rejection",
runtime: { id: "acpx", cloudPlacementSupported: false, source: "model" as const },
executionMode: "worker-turn" as const,
expected: "The acpx runtime does not support cloud workers.",
},
])("$name", ({ runtime, executionMode, executionModes, expected }) => {
])("$name", ({ runtime, executionModes, expected }) => {
const profile: DraftCloudProfile = {
id: "aws",
providerId: "crabbox",
...(executionMode ? { executionMode } : {}),
...(executionModes === undefined ? {} : { executionModes }),
};
const control = new NewSessionModelControl(() => undefined);