diff --git a/docs/gateway/cloud-workers.md b/docs/gateway/cloud-workers.md index 61c83b0949ec..f56057fab72b 100644 --- a/docs/gateway/cloud-workers.md +++ b/docs/gateway/cloud-workers.md @@ -138,6 +138,17 @@ OpenClaw derives one canonical `cbx_...` lease ID from the durable provision ope `settings.setup` runs on the leased box after Crabbox reports it ready and before ephemeral node enrollment. It runs on **every** provision attempt, including replay after an interrupted dispatch, so it must be idempotent — guard installs with a `command -v`/`test -x` check as in the example. At minimum, the resulting machine needs Node.js and `npx`. If setup or enrollment fails, the provider stops the lease and the dispatch fails closed; no half-configured paid box is hidden behind terminal state. +The example profile above prepares OpenClaw worker turns only. To make the same profile Codex-ready, run the following on the Gateway host. It preserves the existing setup, captures the Gateway version, installs that exact OpenClaw release on each cloud node, and installs the matching Codex plugin through the trusted official npm path: + +```bash +gateway_version="$(openclaw --version | awk '{print $2}')" +existing_setup="$(openclaw config get cloudWorkers.profiles.aws.settings.setup)" +openclaw config set cloudWorkers.profiles.aws.settings.setup \ + "${existing_setup}; sudo npm install -g openclaw@${gateway_version}; openclaw plugins install npm:@openclaw/codex@${gateway_version} --pin --force" +``` + +`--force` allows setup replay to converge when the plugin is already installed. After upgrading the Gateway, update both appended package versions to match it while preserving the rest of the setup. Unreleased source builds require exact locally packed packages instead of registry versions. + ### Bundle installation The setup bootstrap first reuses an installed `openclaw` binary when its version exactly matches the Gateway, then tries the exact `openclaw@` registry package. For an unreleased source build, install a locally packed candidate with the same version in `settings.setup`; the provider will select it before touching the registry. After the node connects and publishes its session-host inventory, the Gateway pushes one content-addressed worker bundle through the paired channel. The node verifies and publishes those exact bytes without installing the normal OpenClaw package dependency tree. A stale Gateway build retires the environment and reprovisions against the current bundle rather than downgrading the execution-context protocol. @@ -355,7 +366,7 @@ The desktop never gains public ingress. The node reads `/var/lib/crabbox/vnc.pas - **No cloud profile is advertised** — run the `operator.read`-scoped `openclaw gateway call environments.list --params '{}'`. If the response has no `profiles`, ask an administrator to validate `cloudWorkers.profiles`, inspect the provider plugin, and restart the Gateway. This is a configuration or provider-activation problem, not an authorization result. - **Cloud destinations are hidden or an RPC is denied** — cloud profile dispatch and profile-target moves require `operator.admin`. `operator.write` can dispatch or move to an eligible paired device, move to the Gateway, and reclaim a placement; `operator.read` alone can discover profiles but cannot start, stop, or move a session. Profile configuration, infrastructure pairing, Connect machine, raw environment lifecycle, direct `execNode` execution, incognito sessions, and arbitrary host or node paths remain `operator.admin`. - **The selected runtime lacks cloud placement support** — choose a model whose advertised runtime supports cloud placement. The bundled OpenClaw and Codex runtimes are supported; undeclared runtimes remain local-only. -- **Codex cannot use a cloud profile** — verify that the profile advertises `remote-exec`, the Gateway enables the Codex plugin, and `gateway.nodes.commands.allow` includes `codex.exec-server.stdio.v1` without a matching deny rule. Approve the exact node invocation when prompted. Codex does not require an available OpenClaw worker slot; a missing or denied command must be corrected rather than bypassed with Gateway or SSH execution. +- **Codex cannot use a cloud profile** — verify that the profile advertises `remote-exec`, its setup installs the exact official `npm:@openclaw/codex@` plugin, the Gateway enables the Codex plugin, and `gateway.nodes.commands.allow` includes `codex.exec-server.stdio.v1` without a matching deny rule. Approve the exact node invocation when prompted. Codex does not require an available OpenClaw worker slot; a missing plugin or denied command must be corrected rather than bypassed with Gateway or SSH execution. - **"Worker bootstrap requires Node.js on the leased host"** — add a Node install to `settings.setup` (see above). - **AWS instance-role attestation fails** — clear `aws.instanceProfile` (and `CRABBOX_AWS_INSTANCE_PROFILE`, if set). Install Crabbox 0.41.1 or newer; older binaries do not satisfy the fixed-ID and authoritative `providerMetadata.instanceProfileAttached` contracts required for AWS admission. - **Dispatch or workspace recovery fails** — inspect `environments.list` and `sessions.describe`. A failed environment exposes its bounded environment error. A failed placement exposes `recoveryError` plus its durable per-session `terminalReason`; the selected Control UI chat shows that terminal reason above the composer. When deeper diagnosis is necessary, an operator on the Gateway host can inspect the durable worker state read-only. Do not edit the state database to bypass lifecycle fencing. diff --git a/extensions/crabbox/src/crabbox-worker-node-enrollment.ts b/extensions/crabbox/src/crabbox-worker-node-enrollment.ts index d8ad34bb578c..47788f480426 100644 --- a/extensions/crabbox/src/crabbox-worker-node-enrollment.ts +++ b/extensions/crabbox/src/crabbox-worker-node-enrollment.ts @@ -38,7 +38,7 @@ export function createCrabboxNodeEnrollmentSetup(params: { enrollment.mode === "connect" ? `connect --target-file "$setup_code_file" --ephemeral --display-name ${shellQuote(enrollment.displayName)}` : `node run --ephemeral --display-name ${shellQuote(enrollment.displayName)}`; - const prepareCodex = (binary: string): string[] => { + const prepareCodex = (): string[] => { if (executionMode !== "remote-exec") { return []; } @@ -71,8 +71,8 @@ export function createCrabboxNodeEnrollmentSetup(params: { "}", ].join(""); return [ - ` ${binary} plugins inspect codex --json | node -e ${shellQuote(inspectPlugin)} "$state_dir"`, - ` OPENCLAW_STATE_DIR="$state_dir" ${binary} plugins enable codex`, + `"$@" plugins inspect codex --json | node -e ${shellQuote(inspectPlugin)} "$state_dir"`, + 'OPENCLAW_STATE_DIR="$state_dir" "$@" plugins enable codex', ]; }; const command = [ @@ -106,12 +106,12 @@ export function createCrabboxNodeEnrollmentSetup(params: { "fi", 'package_spec="$(cat "$package_spec_file")"', 'if [ "$package_spec" = "@global" ]; then', - ...prepareCodex("openclaw"), - ` setsid -f sh -c 'printf "%s\\n" "$$" >"$1"; shift; exec "$@"' sh "$pid_file" env OPENCLAW_STATE_DIR="$state_dir" openclaw ${launch} >"$state_dir/node.log" 2>&1 "$1"; shift; exec "$@"' sh "$pid_file" env OPENCLAW_STATE_DIR="$state_dir" npx --yes --package "$package_spec" -- openclaw ${launch} >"$state_dir/node.log" 2>&1 "$1"; shift; exec "$@"' sh "$pid_file" env OPENCLAW_STATE_DIR="$state_dir" "$@" ${launch} >"$state_dir/node.log" 2>&1 { expect(setup).toContain('[launcher,"--version"]'); expect(setup).toContain("codex-cli ${runtime.version}"); expect(setup).not.toContain("$state_dir/extensions/codex"); - expect(setup).toContain('OPENCLAW_STATE_DIR="$state_dir" openclaw plugins enable codex'); - expect(setup).toContain( - 'OPENCLAW_STATE_DIR="$state_dir" npx --yes --package "$package_spec" -- openclaw plugins enable codex', - ); + expect(setup).toContain("set -- openclaw"); + expect(setup).toContain('set -- npx --yes --package "$package_spec" -- openclaw'); + expect(setup).toContain('OPENCLAW_STATE_DIR="$state_dir" "$@" plugins enable codex'); + expect(setup.match(/plugins inspect codex --json/g)).toHaveLength(1); + expect(setup.match(/setsid -f sh -c/g)).toHaveLength(1); expect(setup.indexOf("plugins inspect codex --json")).toBeGreaterThan( setup.indexOf(setupCodeCleared), ); diff --git a/src/gateway/worker-environments/provider-bootstrap.test.ts b/src/gateway/worker-environments/provider-bootstrap.test.ts index 39cd63ee721f..e63d26ab41e8 100644 --- a/src/gateway/worker-environments/provider-bootstrap.test.ts +++ b/src/gateway/worker-environments/provider-bootstrap.test.ts @@ -15,10 +15,12 @@ describe("worker environment service", () => { support.setupWorkerEnvironmentServiceSuite(); it("fails node provisioning visibly when Gateway bundle installation fails", async () => { + const destroy = vi.fn(async () => {}); const workerService = support.createService( support.createProvider({ supportedExecutionModes: ["worker-turn"], provisionBeforeInstallation: true, + destroy, provision: async () => ({ leaseId: "device-lease-install-failure", node: { deviceId: "device-1" }, @@ -35,10 +37,62 @@ describe("worker environment service", () => { workerService.create("development", "request-device-install-failure"), ).rejects.toMatchObject({ code: "bootstrap_failure", - message: expect.stringContaining("bundle transfer unavailable"), + message: "Worker node bootstrap failed: bundle transfer unavailable", } satisfies Partial); + expect(destroy).toHaveBeenCalledWith({ + leaseId: "device-lease-install-failure", + profile: { region: "test" }, + }); expect(support.testState.store.list()[0]).toMatchObject({ state: "failed", + leaseId: null, + nodeDeviceId: null, + lastError: expect.stringContaining("bundle transfer unavailable"), + }); + }); + + it("keeps an indeterminate node bootstrap teardown retryable", async () => { + let teardownFails = true; + const workerService = support.createService( + support.createProvider({ + supportedExecutionModes: ["worker-turn"], + provisionBeforeInstallation: true, + provision: async () => ({ + leaseId: "device-lease-cleanup-failure", + node: { deviceId: "device-cleanup-failure" }, + }), + destroy: async () => { + if (teardownFails) { + throw new Error("provider teardown timed out"); + } + }, + }), + { + ensureNodeWorkerBundle: async () => { + throw new Error("bundle transfer unavailable"); + }, + }, + ); + + await expect( + workerService.create("development", "request-device-cleanup"), + ).rejects.toMatchObject({ + code: "bootstrap_failure", + message: "Worker node bootstrap failed; teardown is pending: bundle transfer unavailable", + } satisfies Partial); + expect(support.testState.store.list()[0]).toMatchObject({ + state: "destroying", + leaseId: "device-lease-cleanup-failure", + nodeDeviceId: "device-cleanup-failure", + teardownTerminalState: "failed", + }); + + teardownFails = false; + await workerService.reconcileOnce(); + expect(support.testState.store.list()[0]).toMatchObject({ + state: "failed", + leaseId: null, + nodeDeviceId: null, lastError: expect.stringContaining("bundle transfer unavailable"), }); }); diff --git a/src/gateway/worker-environments/provider-lifecycle.ts b/src/gateway/worker-environments/provider-lifecycle.ts index c972f4452c6c..b035a9b8b213 100644 --- a/src/gateway/worker-environments/provider-lifecycle.ts +++ b/src/gateway/worker-environments/provider-lifecycle.ts @@ -136,7 +136,9 @@ export function createWorkerProviderLifecycle(options: WorkerProviderLifecycleOp const failureLabel = failureCode === "invalid_profile" ? "Worker provider returned an incompatible lease" - : "Worker bootstrap failed"; + : leasePatch?.nodeDeviceId + ? "Worker node bootstrap failed" + : "Worker bootstrap failed"; const requested = store.requestDestroy({ environmentId: record.environmentId, state: record.state, @@ -184,18 +186,10 @@ export function createWorkerProviderLifecycle(options: WorkerProviderLifecycleOp }; const finishNodeProvisioning = createWorkerNodeProvisioning({ - store, - tunnels, ensureNodeWorkerBundle: options.ensureNodeWorkerBundle, commitReady, - move, - destroyProviderLease: async (record, leaseId, provider) => - await callProvider(record.environmentId, () => - provider.destroy(lifecycleLease(record, leaseId)), - ), - finishProvenDestroy, - saveError, - serviceError, + failBootstrap: async (record, leaseId, provider, error, patch) => + await failBootstrap(record, leaseId, provider, error, "bootstrap_failure", patch), }); const finishBootstrap = async ( diff --git a/src/gateway/worker-environments/provider-node-provisioning.ts b/src/gateway/worker-environments/provider-node-provisioning.ts index 5c4e604eac18..cabbe996ef97 100644 --- a/src/gateway/worker-environments/provider-node-provisioning.ts +++ b/src/gateway/worker-environments/provider-node-provisioning.ts @@ -1,78 +1,34 @@ import type { WorkerAdmissionHandshake } from "../../../packages/gateway-protocol/src/schema/worker-admission.js"; import type { WorkerLease, WorkerProvider } from "../../plugins/types.js"; import type { WorkerCredentialBroker } from "./credential-broker.js"; -import type { WorkerEnvironmentRecord, WorkerEnvironmentStore } from "./store.js"; -import type { WorkerTunnelManager } from "./tunnel.js"; -import { boundedWorkerError as boundedError } from "./worker-error.js"; +import type { WorkerEnvironmentRecord, WorkerEnvironmentTransitionPatch } from "./store.js"; type NodeLease = Extract; type WorkerNodeProvisioningOptions = { - store: WorkerEnvironmentStore; - tunnels?: Pick; ensureNodeWorkerBundle?: (deviceId: string) => Promise; commitReady: WorkerCredentialBroker["commitReady"]; - move: ( - record: WorkerEnvironmentRecord, - to: "draining" | "destroying", - patch?: Parameters[0]["patch"], - ) => WorkerEnvironmentRecord; - destroyProviderLease: ( + failBootstrap: ( record: WorkerEnvironmentRecord, leaseId: string, provider: WorkerProvider, - ) => Promise; - finishProvenDestroy: (record: WorkerEnvironmentRecord) => Promise; - saveError: (record: WorkerEnvironmentRecord, error: unknown) => WorkerEnvironmentRecord; - serviceError: (code: "bootstrap_failure", message: string) => Error; + error: unknown, + patch: WorkerEnvironmentTransitionPatch, + ) => Promise; }; export function createWorkerNodeProvisioning(options: WorkerNodeProvisioningOptions) { - const fail = async ( - record: WorkerEnvironmentRecord, - lease: NodeLease, - provider: WorkerProvider, - error: unknown, - ): Promise => { - const detail = boundedError(error); - const requested = options.store.requestDestroy({ - environmentId: record.environmentId, - state: record.state, - terminalState: "failed", - lastError: detail, - }); - const draining = options.move(requested, "draining", { - leaseId: lease.leaseId, - nodeDeviceId: lease.node.deviceId, - sshEndpoint: null, - sharedHost: lease.sharedHost === true, - desktop: lease.desktop ?? null, - lastError: detail, - }); - await options.tunnels?.stop(record.environmentId); - const destroying = options.move(draining, "destroying", { lastError: detail }); - try { - await options.destroyProviderLease(record, lease.leaseId, provider); - } catch (cleanupError: unknown) { - options.saveError( - destroying, - new Error(`${detail}; provider teardown pending: ${boundedError(cleanupError)}`), - ); - throw options.serviceError( - "bootstrap_failure", - `Worker node bootstrap failed; teardown is pending: ${detail}`, - ); - } - await options.finishProvenDestroy(destroying); - throw options.serviceError("bootstrap_failure", `Worker node bootstrap failed: ${detail}`); - }; - return async ( record: WorkerEnvironmentRecord, lease: NodeLease, provider: WorkerProvider, patch: { leaseId: string; sharedHost: boolean; desktop: WorkerLease["desktop"] | null }, ): Promise => { + const nodePatch = { + ...patch, + nodeDeviceId: lease.node.deviceId, + sshEndpoint: null, + }; let nodeBuild: WorkerAdmissionHandshake; try { if (!options.ensureNodeWorkerBundle) { @@ -80,16 +36,8 @@ export function createWorkerNodeProvisioning(options: WorkerNodeProvisioningOpti } nodeBuild = await options.ensureNodeWorkerBundle(lease.node.deviceId); } catch (error) { - return await fail(record, lease, provider, error); + return await options.failBootstrap(record, lease.leaseId, provider, error, nodePatch); } - return options.commitReady( - record, - { ...nodeBuild, installKind: "bundle" }, - { - ...patch, - nodeDeviceId: lease.node.deviceId, - sshEndpoint: null, - }, - ); + return options.commitReady(record, { ...nodeBuild, installKind: "bundle" }, nodePatch); }; } diff --git a/src/gateway/worker-environments/service.ts b/src/gateway/worker-environments/service.ts index e9b8c9d0f945..7e2e9df8e7b4 100644 --- a/src/gateway/worker-environments/service.ts +++ b/src/gateway/worker-environments/service.ts @@ -520,8 +520,8 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService } return environmentAccess.project( await providerLifecycle.createWithProfile(profileId, idempotencyKey, { - ...(machineClass === undefined ? {} : { machineClass }), - ...(executionMode === undefined ? {} : { executionMode }), + machineClass, + executionMode, }), ); }, @@ -538,8 +538,8 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService providerId: profile.providerId, profileSnapshot: profile.profileSnapshot, }, - ...(machineClass === undefined ? {} : { machineClass }), - ...(executionMode === undefined ? {} : { executionMode }), + machineClass, + executionMode, }), ); }, diff --git a/ui/src/pages/new-session/discovery.test.ts b/ui/src/pages/new-session/discovery.test.ts index 0e306c78a3ec..49229077ff96 100644 --- a/ui/src/pages/new-session/discovery.test.ts +++ b/ui/src/pages/new-session/discovery.test.ts @@ -23,12 +23,10 @@ describe("draftCloudProfileSupportsExecutionMode", () => { ); it("does not treat the singular display projection as a placement capability", () => { - expect( - draftCloudProfileSupportsExecutionMode( - { id: "legacy", providerId: "crabbox", executionMode: "worker-turn" }, - "worker-turn", - ), - ).toBe(false); + const [profile] = readDraftCloudProfiles([ + { id: "legacy", providerId: "crabbox", executionMode: "worker-turn" }, + ]); + expect(draftCloudProfileSupportsExecutionMode(profile!, "worker-turn")).toBe(false); }); }); @@ -79,7 +77,6 @@ describe("readDraftCloudProfiles", () => { id: "aws", providerId: "crabbox", trust: "persistent", - executionMode: "worker-turn", executionModes: ["worker-turn", "remote-exec"], machines: [ { @@ -96,19 +93,16 @@ describe("readDraftCloudProfiles", () => { id: "invalid-trust", providerId: "crabbox", trust: undefined, - executionMode: undefined, }, { id: "legacy", providerId: "static-ssh", trust: undefined, - executionMode: undefined, }, { id: "zeta", providerId: "static-ssh", trust: "disposable", - executionMode: "worker-turn", }, ]); }); @@ -136,7 +130,6 @@ describe("readDraftCloudProfiles", () => { id: "aws", providerId: "crabbox", trust: undefined, - executionMode: "remote-exec", executionModes: [], }, ]); diff --git a/ui/src/pages/new-session/discovery.ts b/ui/src/pages/new-session/discovery.ts index 28c496cdda8e..1a57c0d21052 100644 --- a/ui/src/pages/new-session/discovery.ts +++ b/ui/src/pages/new-session/discovery.ts @@ -29,7 +29,6 @@ export type DraftCloudProfile = { id: string; providerId: string; trust?: "persistent" | "disposable"; - executionMode?: WorkerExecutionMode; executionModes?: readonly WorkerExecutionMode[]; machines?: DraftMachineOption[]; }; @@ -118,7 +117,6 @@ export function readDraftCloudProfiles(value: unknown): DraftCloudProfile[] { id?: unknown; providerId?: unknown; trust?: unknown; - executionMode?: unknown; executionModes?: unknown; machines?: unknown; }; @@ -131,17 +129,12 @@ export function readDraftCloudProfiles(value: unknown): DraftCloudProfile[] { profile.trust === "persistent" || profile.trust === "disposable" ? profile.trust : undefined; - const executionMode: WorkerExecutionMode | undefined = - profile.executionMode === "worker-turn" || profile.executionMode === "remote-exec" - ? profile.executionMode - : undefined; const machines = readDraftMachineOptions(profile.machines); return [ { id, providerId, trust, - executionMode, ...(Object.hasOwn(profile, "executionModes") ? { executionModes: readDraftCloudProfileExecutionModes(profile.executionModes) } : {}), diff --git a/ui/src/pages/new-session/draft-place-state.test.ts b/ui/src/pages/new-session/draft-place-state.test.ts index 59eab9f2ddf2..9bc79770ab4b 100644 --- a/ui/src/pages/new-session/draft-place-state.test.ts +++ b/ui/src/pages/new-session/draft-place-state.test.ts @@ -122,7 +122,6 @@ describe("DraftPlaceState cloud machine selection", () => { { id: "aws", providerId: "crabbox", - executionMode: "worker-turn", executionModes, machines: [ { id: "standard", label: "Standard", default: true }, diff --git a/ui/src/pages/new-session/model-control.runtime-placement.test.ts b/ui/src/pages/new-session/model-control.runtime-placement.test.ts index e7dbcf327488..6ce6e18eba7f 100644 --- a/ui/src/pages/new-session/model-control.runtime-placement.test.ts +++ b/ui/src/pages/new-session/model-control.runtime-placement.test.ts @@ -14,7 +14,6 @@ describe("new-session model runtime placement", () => { cloudPlacementExecutionMode: "remote-exec" as const, source: "model" as const, }, - executionMode: "worker-turn" as const, executionModes: ["worker-turn"] as const, expected: "The codex runtime cannot use this cloud worker. Choose a compatible cloud worker or run locally.", @@ -27,7 +26,6 @@ describe("new-session model runtime placement", () => { cloudPlacementExecutionMode: "worker-turn" as const, source: "model" as const, }, - executionMode: "worker-turn" as const, executionModes: ["worker-turn"] as const, expected: undefined, }, @@ -39,19 +37,17 @@ describe("new-session model runtime placement", () => { cloudPlacementExecutionMode: "worker-turn" as const, source: "model" as const, }, - executionMode: "worker-turn" as const, executionModes: ["worker-turn", "remote-exec"] as const, expected: undefined, }, { - name: "accepts a remote-exec runtime on the same worker-turn-primary profile", + name: "accepts a remote-exec runtime on the same dual-mode profile", runtime: { id: "codex", cloudPlacementSupported: true, cloudPlacementExecutionMode: "remote-exec" as const, source: "model" as const, }, - executionMode: "worker-turn" as const, executionModes: ["worker-turn", "remote-exec"] as const, expected: undefined, }, @@ -63,20 +59,18 @@ describe("new-session model runtime placement", () => { cloudPlacementExecutionMode: "remote-exec" as const, source: "model" as const, }, - executionMode: "worker-turn" as const, executionModes: ["worker-turn"] as const, expected: "The codex runtime cannot use this cloud worker. Choose a compatible cloud worker or run locally.", }, { - name: "does not fall back to the primary mode when an explicit mode set is empty", + name: "rejects an empty placement mode set", runtime: { id: "codex", cloudPlacementSupported: true, cloudPlacementExecutionMode: "remote-exec" as const, source: "model" as const, }, - executionMode: "remote-exec" as const, executionModes: [] as const, expected: "The codex runtime cannot use this cloud worker. Choose a compatible cloud worker or run locally.", @@ -89,21 +83,18 @@ describe("new-session model runtime placement", () => { cloudPlacementExecutionMode: "remote-exec" as const, source: "model" as const, }, - executionMode: undefined, expected: "The codex runtime cannot use this cloud worker. Choose a compatible cloud worker or run locally.", }, { name: "retains the existing whole-runtime rejection", runtime: { id: "acpx", cloudPlacementSupported: false, source: "model" as const }, - executionMode: "worker-turn" as const, expected: "The acpx runtime does not support cloud workers.", }, - ])("$name", ({ runtime, executionMode, executionModes, expected }) => { + ])("$name", ({ runtime, executionModes, expected }) => { const profile: DraftCloudProfile = { id: "aws", providerId: "crabbox", - ...(executionMode ? { executionMode } : {}), ...(executionModes === undefined ? {} : { executionModes }), }; const control = new NewSessionModelControl(() => undefined);