From 2d6a5d7356bb96d1fdb810e97156b2645426f355 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 11 Aug 2026 11:48:28 -0700 Subject: [PATCH] refactor(gateway): split worker environment service by ownership (#122138) * refactor(gateway): split worker environment service by ownership * refactor(gateway): derive environment intent at the contract owner * chore(lint): ratchet max-lines baseline after worker-service split * style(gateway): prune unused symbols after worker-service split * refactor(gateway): narrow worker-service seam exports * chore(plugin-sdk): refresh API baseline --- config/max-lines-baseline.txt | 2 - .../agent-harness-runtime.json | 2 +- .../agent-harness.json | 2 +- .../plugin-sdk-api-baseline/channel-core.json | 2 +- .../channel-entry-contract.json | 2 +- .../channel-message.json | 2 +- .../channel-outbound.json | 2 +- .../channel-plugin-common.json | 2 +- .../plugin-sdk-api-baseline/core.json | 2 +- .../plugin-sdk-api-baseline/discord.json | 2 +- .../inbound-reply-dispatch.json | 2 +- .../meeting-runtime.json | 2 +- .../plugin-sdk-api-baseline/plugin-entry.json | 2 +- .../plugin-runtime.json | 2 +- .../provider-catalog-runtime.json | 2 +- .../plugin-sdk-api-baseline/tool-plugin.json | 2 +- .../webhook-ingress.json | 2 +- ...ver.sessions.worker-original-order.test.ts | 4 +- .../credential-broker.test.ts | 226 ++ .../worker-environments/credential-broker.ts | 351 ++ .../environment-access.test.ts | 440 +++ .../worker-environments/environment-access.ts | 347 ++ .../placement-dispatch-test-fixtures.ts | 6 +- .../placement-dispatch.test.ts | 6 +- .../worker-environments/placement-dispatch.ts | 5 +- .../placement-worker-gate.ts | 29 +- .../worker-environments/provider-lifecycle.ts | 661 ++++ .../provider-provisioning.test.ts | 868 +++++ .../provider-reconciliation.test.ts | 613 ++++ .../worker-environments/service-contract.ts | 12 + .../service-lifetime.test.ts | 226 ++ .../service.test-support.ts | 458 +++ .../worker-environments/service.test.ts | 3165 ----------------- src/gateway/worker-environments/service.ts | 1948 +--------- .../worker-turn-rpc.test.ts | 563 +++ .../worker-environments/worker-turn-rpc.ts | 597 ++++ 36 files changed, 5582 insertions(+), 4977 deletions(-) create mode 100644 src/gateway/worker-environments/credential-broker.test.ts create mode 100644 src/gateway/worker-environments/credential-broker.ts create mode 100644 src/gateway/worker-environments/environment-access.test.ts create mode 100644 src/gateway/worker-environments/environment-access.ts create mode 100644 src/gateway/worker-environments/provider-lifecycle.ts create mode 100644 src/gateway/worker-environments/provider-provisioning.test.ts create mode 100644 src/gateway/worker-environments/provider-reconciliation.test.ts create mode 100644 src/gateway/worker-environments/service-lifetime.test.ts create mode 100644 src/gateway/worker-environments/service.test-support.ts delete mode 100644 src/gateway/worker-environments/service.test.ts create mode 100644 src/gateway/worker-environments/worker-turn-rpc.test.ts create mode 100644 src/gateway/worker-environments/worker-turn-rpc.ts diff --git a/config/max-lines-baseline.txt b/config/max-lines-baseline.txt index b53bf36bdd22..89a72e57fa2c 100644 --- a/config/max-lines-baseline.txt +++ b/config/max-lines-baseline.txt @@ -732,8 +732,6 @@ src/gateway/tools-invoke-http.test.ts src/gateway/watch-node-http.ts src/gateway/worker-environments/bootstrap.ts src/gateway/worker-environments/inference-runtime.ts -src/gateway/worker-environments/service.test.ts -src/gateway/worker-environments/service.ts src/gateway/worker-environments/store.ts src/gateway/worker-environments/worker-turn-launcher.test.ts src/hooks/install.ts diff --git a/docs/.generated/plugin-sdk-api-baseline/agent-harness-runtime.json b/docs/.generated/plugin-sdk-api-baseline/agent-harness-runtime.json index 380be9440ef7..9857b3eab9b3 100644 --- a/docs/.generated/plugin-sdk-api-baseline/agent-harness-runtime.json +++ b/docs/.generated/plugin-sdk-api-baseline/agent-harness-runtime.json @@ -1 +1 @@ -{"contentHash":"66cc8d088b99fc543585ebc69c2149ec31cde693d580926faeeb56a756002ce0","entrypoint":"agent-harness-runtime","importSpecifier":"openclaw/plugin-sdk/agent-harness-runtime"} +{"contentHash":"935472a542772d2a003ced8e4ff4b21082d4a7ceb87162605eb3f081f3134c9e","entrypoint":"agent-harness-runtime","importSpecifier":"openclaw/plugin-sdk/agent-harness-runtime"} diff --git a/docs/.generated/plugin-sdk-api-baseline/agent-harness.json b/docs/.generated/plugin-sdk-api-baseline/agent-harness.json index 1b75e5f4e97a..24eefbb73de6 100644 --- a/docs/.generated/plugin-sdk-api-baseline/agent-harness.json +++ b/docs/.generated/plugin-sdk-api-baseline/agent-harness.json @@ -1 +1 @@ -{"contentHash":"98f0954479848e96da80f4fe4dec84580ea02eb3c99c4bfb8b8650f560852b8a","entrypoint":"agent-harness","importSpecifier":"openclaw/plugin-sdk/agent-harness"} +{"contentHash":"3012ea8e6338549d7523326c5cf006e0cf6bcaef8cf0cfbd3b36001d18a00a79","entrypoint":"agent-harness","importSpecifier":"openclaw/plugin-sdk/agent-harness"} diff --git a/docs/.generated/plugin-sdk-api-baseline/channel-core.json b/docs/.generated/plugin-sdk-api-baseline/channel-core.json index a9c820967c32..0bf2ea8934ed 100644 --- a/docs/.generated/plugin-sdk-api-baseline/channel-core.json +++ b/docs/.generated/plugin-sdk-api-baseline/channel-core.json @@ -1 +1 @@ -{"contentHash":"f4818fe0f464f65b8c564d6d14ef0867c7ad933a888827acf97944c18789a15c","entrypoint":"channel-core","importSpecifier":"openclaw/plugin-sdk/channel-core"} +{"contentHash":"dee0293611efcd65fa6ada3b091a8cd4cc4f9fc08322ee09f66b86d06c7c0da7","entrypoint":"channel-core","importSpecifier":"openclaw/plugin-sdk/channel-core"} diff --git a/docs/.generated/plugin-sdk-api-baseline/channel-entry-contract.json b/docs/.generated/plugin-sdk-api-baseline/channel-entry-contract.json index df16308996ec..66e2c65b5d45 100644 --- a/docs/.generated/plugin-sdk-api-baseline/channel-entry-contract.json +++ b/docs/.generated/plugin-sdk-api-baseline/channel-entry-contract.json @@ -1 +1 @@ -{"contentHash":"c704b85a419f1d4cb581c4a17e8d30ad2893cd1e177f46245c33e8b83be70edf","entrypoint":"channel-entry-contract","importSpecifier":"openclaw/plugin-sdk/channel-entry-contract"} +{"contentHash":"db8f468e891f92ca526beea61a7d31aa4ec3c274869418a97f107e880b3b0d43","entrypoint":"channel-entry-contract","importSpecifier":"openclaw/plugin-sdk/channel-entry-contract"} diff --git a/docs/.generated/plugin-sdk-api-baseline/channel-message.json b/docs/.generated/plugin-sdk-api-baseline/channel-message.json index 6c7a8243fcca..e95917b67e51 100644 --- a/docs/.generated/plugin-sdk-api-baseline/channel-message.json +++ b/docs/.generated/plugin-sdk-api-baseline/channel-message.json @@ -1 +1 @@ -{"contentHash":"0d442cf852cb35129f2eccfee5c4031946404fc691bb3690b7aa54dd4c7d550d","entrypoint":"channel-message","importSpecifier":"openclaw/plugin-sdk/channel-message"} +{"contentHash":"76a2a221a6a6f2295037d6ee58cac1d82adac001c6dc0467724c41ca1487d555","entrypoint":"channel-message","importSpecifier":"openclaw/plugin-sdk/channel-message"} diff --git a/docs/.generated/plugin-sdk-api-baseline/channel-outbound.json b/docs/.generated/plugin-sdk-api-baseline/channel-outbound.json index 7cfc3cfae364..cfd278b733c7 100644 --- a/docs/.generated/plugin-sdk-api-baseline/channel-outbound.json +++ b/docs/.generated/plugin-sdk-api-baseline/channel-outbound.json @@ -1 +1 @@ -{"contentHash":"3a937d8039d8ebe37f71b7fce57251adc0e3eb9c5aa6b138cd695a39350b1991","entrypoint":"channel-outbound","importSpecifier":"openclaw/plugin-sdk/channel-outbound"} +{"contentHash":"a288e638fab5e190ba81e2312d52a2fb4fbbfe09a764ce8ae71aead70c1a1f22","entrypoint":"channel-outbound","importSpecifier":"openclaw/plugin-sdk/channel-outbound"} diff --git a/docs/.generated/plugin-sdk-api-baseline/channel-plugin-common.json b/docs/.generated/plugin-sdk-api-baseline/channel-plugin-common.json index 4c8c6894d727..15a4c72d7de6 100644 --- a/docs/.generated/plugin-sdk-api-baseline/channel-plugin-common.json +++ b/docs/.generated/plugin-sdk-api-baseline/channel-plugin-common.json @@ -1 +1 @@ -{"contentHash":"3965e6960af8ac5f36f2132c218b5c38e4aa5274c96bc2e659ec2929d137beca","entrypoint":"channel-plugin-common","importSpecifier":"openclaw/plugin-sdk/channel-plugin-common"} +{"contentHash":"fc7650d41aa0ee4feeb7d4a8c624f00666e5a280c0e400c2637dc4c90639f406","entrypoint":"channel-plugin-common","importSpecifier":"openclaw/plugin-sdk/channel-plugin-common"} diff --git a/docs/.generated/plugin-sdk-api-baseline/core.json b/docs/.generated/plugin-sdk-api-baseline/core.json index e8b2bb960fa8..8737f0d2b568 100644 --- a/docs/.generated/plugin-sdk-api-baseline/core.json +++ b/docs/.generated/plugin-sdk-api-baseline/core.json @@ -1 +1 @@ -{"contentHash":"4d030c8d32ee935043063a15692eba9e15020182144f9770a76b520351d53b9e","entrypoint":"core","importSpecifier":"openclaw/plugin-sdk/core"} +{"contentHash":"b7ef17f9a6a055f6ccc6082ecfad2b3e14e54eb2538d8bcae2b14d48c38efc02","entrypoint":"core","importSpecifier":"openclaw/plugin-sdk/core"} diff --git a/docs/.generated/plugin-sdk-api-baseline/discord.json b/docs/.generated/plugin-sdk-api-baseline/discord.json index 9e1bb43352bd..24f9eb22e1cb 100644 --- a/docs/.generated/plugin-sdk-api-baseline/discord.json +++ b/docs/.generated/plugin-sdk-api-baseline/discord.json @@ -1 +1 @@ -{"contentHash":"a3d10effa80491699a93aabbc6cb3ac404913fde87b52a9566f1360a679a4e03","entrypoint":"discord","importSpecifier":"openclaw/plugin-sdk/discord"} +{"contentHash":"13b35ecb4af318bc7688c39b790fa90c145886358e3256cfe24a70412a20e70f","entrypoint":"discord","importSpecifier":"openclaw/plugin-sdk/discord"} diff --git a/docs/.generated/plugin-sdk-api-baseline/inbound-reply-dispatch.json b/docs/.generated/plugin-sdk-api-baseline/inbound-reply-dispatch.json index c4cc8d6a42a0..a494515a345e 100644 --- a/docs/.generated/plugin-sdk-api-baseline/inbound-reply-dispatch.json +++ b/docs/.generated/plugin-sdk-api-baseline/inbound-reply-dispatch.json @@ -1 +1 @@ -{"contentHash":"31e609e0fda8b6561856f788419ca055659a19fbe724397d8e12510408339ca9","entrypoint":"inbound-reply-dispatch","importSpecifier":"openclaw/plugin-sdk/inbound-reply-dispatch"} +{"contentHash":"bc04e864df9f24ba9d6436e27030679e2e8ad0143e6f9f4012118c6526a43395","entrypoint":"inbound-reply-dispatch","importSpecifier":"openclaw/plugin-sdk/inbound-reply-dispatch"} diff --git a/docs/.generated/plugin-sdk-api-baseline/meeting-runtime.json b/docs/.generated/plugin-sdk-api-baseline/meeting-runtime.json index de0d9860f036..2ef4f748d271 100644 --- a/docs/.generated/plugin-sdk-api-baseline/meeting-runtime.json +++ b/docs/.generated/plugin-sdk-api-baseline/meeting-runtime.json @@ -1 +1 @@ -{"contentHash":"65291a0f9266f239812fd732535c0e63f21eb723527e6e930398067b85283607","entrypoint":"meeting-runtime","importSpecifier":"openclaw/plugin-sdk/meeting-runtime"} +{"contentHash":"b4e87ccdd939708f75d8b5807a7faf341ae1be06300cd6ad26b79173d1ba17a6","entrypoint":"meeting-runtime","importSpecifier":"openclaw/plugin-sdk/meeting-runtime"} diff --git a/docs/.generated/plugin-sdk-api-baseline/plugin-entry.json b/docs/.generated/plugin-sdk-api-baseline/plugin-entry.json index d67d94531089..833f8e91c400 100644 --- a/docs/.generated/plugin-sdk-api-baseline/plugin-entry.json +++ b/docs/.generated/plugin-sdk-api-baseline/plugin-entry.json @@ -1 +1 @@ -{"contentHash":"d929e7a1c19c0403bb260610c881e9a98b3ce7b307729d3c40f72b16476a06c9","entrypoint":"plugin-entry","importSpecifier":"openclaw/plugin-sdk/plugin-entry"} +{"contentHash":"83527106b0a04b3f7b812c47001498bd005fd9911f8dd0b97acd35d4d7f61f1f","entrypoint":"plugin-entry","importSpecifier":"openclaw/plugin-sdk/plugin-entry"} diff --git a/docs/.generated/plugin-sdk-api-baseline/plugin-runtime.json b/docs/.generated/plugin-sdk-api-baseline/plugin-runtime.json index 2b1bf571b774..abf3aaefae8f 100644 --- a/docs/.generated/plugin-sdk-api-baseline/plugin-runtime.json +++ b/docs/.generated/plugin-sdk-api-baseline/plugin-runtime.json @@ -1 +1 @@ -{"contentHash":"eef16bc3eca8d5d9cea265e4b09b3cb0a78b08d5238cea9e3df1c6901b46ad37","entrypoint":"plugin-runtime","importSpecifier":"openclaw/plugin-sdk/plugin-runtime"} +{"contentHash":"e326d4d2b59155856361367d5aadebf7962baaebe8ef91155cedb636e0871f92","entrypoint":"plugin-runtime","importSpecifier":"openclaw/plugin-sdk/plugin-runtime"} diff --git a/docs/.generated/plugin-sdk-api-baseline/provider-catalog-runtime.json b/docs/.generated/plugin-sdk-api-baseline/provider-catalog-runtime.json index 1ba79211b9cb..1c9c1684eeba 100644 --- a/docs/.generated/plugin-sdk-api-baseline/provider-catalog-runtime.json +++ b/docs/.generated/plugin-sdk-api-baseline/provider-catalog-runtime.json @@ -1 +1 @@ -{"contentHash":"70229ab9b920b19dd7715d54418294f4071a870d23325b9e6e8623895cdaf5e5","entrypoint":"provider-catalog-runtime","importSpecifier":"openclaw/plugin-sdk/provider-catalog-runtime"} +{"contentHash":"f2d93ae0543dda1166f7f56c6a2f61910dd35ba4f539e4f31a89004020d04e18","entrypoint":"provider-catalog-runtime","importSpecifier":"openclaw/plugin-sdk/provider-catalog-runtime"} diff --git a/docs/.generated/plugin-sdk-api-baseline/tool-plugin.json b/docs/.generated/plugin-sdk-api-baseline/tool-plugin.json index d74cf75edd0e..cbfd0fca157a 100644 --- a/docs/.generated/plugin-sdk-api-baseline/tool-plugin.json +++ b/docs/.generated/plugin-sdk-api-baseline/tool-plugin.json @@ -1 +1 @@ -{"contentHash":"c9e8af2c4fb225217736662fb3b82900523d3e7b1eba766cc1f96d9ef4d2279a","entrypoint":"tool-plugin","importSpecifier":"openclaw/plugin-sdk/tool-plugin"} +{"contentHash":"0fa1302f61d179c11329b49a6d7e1d242f8de7ab73c4e515667343b789a39e7d","entrypoint":"tool-plugin","importSpecifier":"openclaw/plugin-sdk/tool-plugin"} diff --git a/docs/.generated/plugin-sdk-api-baseline/webhook-ingress.json b/docs/.generated/plugin-sdk-api-baseline/webhook-ingress.json index 4a7ab62f9eb3..dfd0d2300d1f 100644 --- a/docs/.generated/plugin-sdk-api-baseline/webhook-ingress.json +++ b/docs/.generated/plugin-sdk-api-baseline/webhook-ingress.json @@ -1 +1 @@ -{"contentHash":"d34db3cba2e1fd488996b6820ba6e422d17472cde8d4f8077704234e8e96d7d8","entrypoint":"webhook-ingress","importSpecifier":"openclaw/plugin-sdk/webhook-ingress"} +{"contentHash":"3d1a41189c7b6a7d0318c9d262737190bdfcf375a7cd12869f90ad906cb2cfcc","entrypoint":"webhook-ingress","importSpecifier":"openclaw/plugin-sdk/webhook-ingress"} diff --git a/src/gateway/server.sessions.worker-original-order.test.ts b/src/gateway/server.sessions.worker-original-order.test.ts index ffe0e49ce544..8cecf6b234e1 100644 --- a/src/gateway/server.sessions.worker-original-order.test.ts +++ b/src/gateway/server.sessions.worker-original-order.test.ts @@ -22,10 +22,10 @@ import { bootstrapWorker } from "./worker-environments/bootstrap.js"; import type { WorkerInstallationArtifact } from "./worker-environments/bundle.js"; import { createWorkerPlacementDispatchService } from "./worker-environments/placement-dispatch.js"; import { createWorkerSessionPlacementStore } from "./worker-environments/placement-store.js"; +import { deriveEnvironmentIntent } from "./worker-environments/service-contract.js"; import { createWorkerEnvironmentService, type WorkerEnvironmentService, - workerEnvironmentIdForIdempotencyKey, } from "./worker-environments/service.js"; import { createWorkerEnvironmentStore } from "./worker-environments/store.js"; import type { WorkerSshProcess, WorkerSshRunner } from "./worker-environments/tunnel-ssh-runner.js"; @@ -40,7 +40,7 @@ const FALLBACK_PORT = 22; const SESSION_ID = "session-original-order"; const SESSION_KEY = "agent:main:original-order"; const PROFILE_ID = "development"; -const ENVIRONMENT_ID = workerEnvironmentIdForIdempotencyKey(`session-dispatch:${SESSION_ID}:1`); +const ENVIRONMENT_ID = deriveEnvironmentIntent(`session-dispatch:${SESSION_ID}:1`).environmentId; const BUNDLE_HASH = "a".repeat(64); const RECEIPT = { bundleHash: BUNDLE_HASH, diff --git a/src/gateway/worker-environments/credential-broker.test.ts b/src/gateway/worker-environments/credential-broker.test.ts new file mode 100644 index 000000000000..387b6584899f --- /dev/null +++ b/src/gateway/worker-environments/credential-broker.test.ts @@ -0,0 +1,226 @@ +import { describe, expect, it, vi } from "vitest"; +import { + closeOpenClawStateDatabaseForTest, + openOpenClawStateDatabase, +} from "../../state/openclaw-state-db.js"; +import * as support from "./service.test-support.js"; +import { createWorkerEnvironmentStore } from "./store.js"; +import type { WorkerTunnelManager } from "./tunnel.js"; + +describe("worker environment service", () => { + support.setupWorkerEnvironmentServiceSuite(); + + it("repairs duplicate session owners", async () => { + const sessionId = "legacy"; + const older = support.seedAttachedIdentity("legacy-a", sessionId); + const newer = support.seedAttachedIdentity("legacy-b", "other"); + support.testState.stateDb.db.exec(` + UPDATE worker_environments SET attached_session_ids_json = '["legacy"]' + WHERE environment_id = 'legacy-b'; + UPDATE worker_environment_credentials SET session_id = 'legacy' + WHERE environment_id = 'legacy-b'; + `); + + closeOpenClawStateDatabaseForTest(); + support.testState.stateDb = openOpenClawStateDatabase({ + env: { OPENCLAW_STATE_DIR: support.testState.root }, + }); + support.testState.store = createWorkerEnvironmentStore({ + database: support.testState.stateDb, + now: () => support.testState.nowMs, + }); + const liveEvents = support.createLiveEvents(); + const workerService = support.createService(support.createProvider(), { liveEvents }); + const event = { ...support.LIVE_EVENT, runEpoch: newer.ownerEpoch }; + await expect(workerService.pushLiveEvent(older, event)).resolves.toEqual({ + ok: false, + closeReason: "credential-replaced", + }); + await workerService.pushLiveEvent({ ...newer, sessionId }, event); + expect(liveEvents.apply).toHaveBeenCalledOnce(); + }); + + it("rejects attach before current bootstrap", async () => { + const staleId = "worker-stale-attach"; + const bootstrapping = support.seedBootstrapping(staleId); + support.testState.store.transition({ + environmentId: staleId, + from: bootstrapping.state, + to: "ready", + patch: support.readyPatch(staleId, { + ...support.BOOTSTRAP_RECEIPT, + bundleHash: "c".repeat(64), + }), + }); + const workerService = support.createService(support.createProvider()); + + await expect( + workerService.attachSession({ + environmentId: staleId, + ownerEpoch: 1, + sessionId: "session-1", + }), + ).rejects.toThrow("must bootstrap the current build"); + expect(support.testState.store.get(staleId)).toMatchObject({ + state: "ready", + attachedSessionIds: [], + }); + }); + + it("returns a bounded error when another worker owns the session", async () => { + const firstId = "worker-session-owner"; + const secondId = "worker-session-contender"; + support.seedReady(firstId); + support.seedReady(secondId); + const workerService = support.createService(support.createProvider()); + + await workerService.attachSession({ + environmentId: firstId, + ownerEpoch: 1, + sessionId: "session-owned", + }); + await expect( + workerService.attachSession({ + environmentId: secondId, + ownerEpoch: 1, + sessionId: "session-owned", + }), + ).rejects.toMatchObject({ + code: "invalid_state", + message: + "Session session-owned is already attached to worker environment worker-session-owner", + }); + expect(support.testState.store.get(secondId)).toMatchObject({ + state: "ready", + attachedSessionIds: [], + }); + }); + + it("requires session reclaim before operator destruction of an attached worker", async () => { + const environmentId = "worker-session-reclaim"; + support.seedReady(environmentId); + const workerService = support.createService(support.createProvider()); + await workerService.attachSession({ + environmentId, + ownerEpoch: 1, + sessionId: "session-reclaim", + }); + + await expect(workerService.destroyUnattached(environmentId)).rejects.toMatchObject({ + code: "invalid_state", + message: "Attached cloud workers must be stopped through sessions.reclaim", + }); + expect(support.testState.store.get(environmentId)).toMatchObject({ + state: "attached", + attachedSessionIds: ["session-reclaim"], + }); + }); + + it("stops the tunnel after live binding rollback", async () => { + const environmentId = "live-bind-fail"; + support.seedReady(environmentId); + const liveEvents = support.createLiveEvents({ + bindSession: vi.fn(() => { + throw new Error("bind failed"); + }), + }); + const tunnelManager = { + stop: vi.fn(async () => {}), + stopAll: vi.fn(async () => {}), + } as unknown as WorkerTunnelManager; + const workerService = support.createService(support.createProvider(), { + liveEvents, + tunnelManager, + }); + + await expect( + workerService.attachSession({ environmentId, ownerEpoch: 1, sessionId: "session-live" }), + ).rejects.toThrow("Attached session target is unavailable"); + expect(tunnelManager.stop).toHaveBeenCalledWith(environmentId, 1); + expect(support.testState.store.get(environmentId)).toMatchObject({ + state: "idle", + attachedSessionIds: [], + }); + }); + + it("renews in place and binds delivery acknowledgement to the exact grant", async () => { + const environmentId = "worker-credential-replacement"; + support.seedReady(environmentId); + let credentialSequence = 0; + const workerService = support.createService(support.createProvider(), { + generateWorkerCredential: () => [support.CREDENTIAL, String(++credentialSequence)].join("-"), + workerCredentialTtlMs: 100, + }); + + const binding = { environmentId, ownerEpoch: 1, sessionId: null }; + await workerService.reconcileOnce(); + const previous = workerService.takeMintedCredential(binding)!; + support.testState.nowMs += 100; + expect(workerService.takeMintedCredential(binding)).toBeUndefined(); + await workerService.reconcileOnce(); + const renewal = workerService.takeMintedCredential(binding)!; + expect(renewal).toMatchObject({ ownerEpoch: 1, sessionId: null }); + expect(support.testState.store.get(environmentId)?.ownerEpoch).toBe(1); + expect(workerService.acknowledgeCredentialDelivery(previous)).toBe(false); + expect(workerService.takeMintedCredential(binding)).toMatchObject({ + deliveryId: renewal.deliveryId, + }); + expect(workerService.acknowledgeCredentialDelivery(renewal)).toBe(true); + }); + + it("recovers an undelivered atomic session credential after restart without changing owner", async () => { + const environmentId = "worker-attach-restart"; + support.seedReady(environmentId); + let credentialSequence = 0; + const stopTunnel = vi.fn(async () => { + throw new Error("tunnel stop interrupted"); + }); + const tunnelManager = { + start: vi.fn(), + stop: stopTunnel, + stopAll: vi.fn(async () => {}), + status: () => "stopped" as const, + desktop: { + acquire: vi.fn(), + attachObserver: vi.fn(), + stop: vi.fn(async () => {}), + stopAll: vi.fn(async () => {}), + }, + } as unknown as WorkerTunnelManager; + const options = { + generateWorkerCredential: () => [support.CREDENTIAL, String(++credentialSequence)].join("-"), + tunnelManager, + }; + const first = support.createService(support.createProvider(), options); + await first.reconcileOnce(); + await expect( + first.attachSession({ environmentId, ownerEpoch: 1, sessionId: "session-1" }), + ).rejects.toThrow("tunnel stop interrupted"); + const binding = { environmentId, ownerEpoch: 2, sessionId: "session-1" }; + const lostHash = support.testState.store.getCredential(environmentId)?.credentialHash; + + expect(stopTunnel).toHaveBeenCalledWith(environmentId, 1); + expect(support.testState.store.get(environmentId)).toMatchObject({ + state: "attached", + ownerEpoch: 2, + }); + expect(first.takeMintedCredential(binding)).toBeUndefined(); + + await first.stop(); + const restarted = support.createService(support.createProvider(), options); + await restarted.reconcileOnce(); + + const recovered = restarted.takeMintedCredential(binding); + expect(recovered?.deliveryId).not.toBe(lostHash); + expect(restarted.acknowledgeCredentialDelivery(recovered!)).toBe(true); + const deliveredHash = support.testState.store.getCredential(environmentId)?.credentialHash; + + await restarted.stop(); + const deliveredRestart = support.createService(support.createProvider(), options); + await deliveredRestart.reconcileOnce(); + expect(deliveredRestart.takeMintedCredential(binding)).toBeUndefined(); + expect(support.testState.store.getCredential(environmentId)?.credentialHash).toBe( + deliveredHash, + ); + }); +}); diff --git a/src/gateway/worker-environments/credential-broker.ts b/src/gateway/worker-environments/credential-broker.ts new file mode 100644 index 000000000000..1270ca142f4b --- /dev/null +++ b/src/gateway/worker-environments/credential-broker.ts @@ -0,0 +1,351 @@ +import { WORKER_RPC_SET_VERSION } from "../../../packages/gateway-protocol/src/schema/worker-admission.js"; +import { verifyWorkerAdmissionHandshake } from "./admission.js"; +import type { WorkerInstallationArtifact } from "./bundle.js"; +import { + createWorkerCredentialMaterial, + WORKER_CREDENTIAL_TTL_MS, + type MintedWorkerCredential, + type WorkerCredentialBinding, + type WorkerCredentialDeliveryClaim, +} from "./credential.js"; +import type { WorkerLiveEventReceiver } from "./live-events.js"; +import type { WorkerSessionPlacementGate } from "./placement-worker-gate.js"; +import type { WorkerEnvironmentState } from "./state.js"; +import { + type WorkerEnvironmentRecord, + type WorkerEnvironmentStore, + WorkerSessionAlreadyAttachedError, +} from "./store.js"; +import type { WorkerTunnelManager } from "./tunnel.js"; + +type WorkerCredentialBrokerOptions = { + store: WorkerEnvironmentStore; + prepareInstallation: ( + install: WorkerInstallationArtifact["install"], + ) => Promise; + tunnelManager?: WorkerTunnelManager; + workerCredentialTtlMs?: number; + generateWorkerCredential?: (bytes: number) => string; + liveEvents?: Pick; + placementStore?: WorkerSessionPlacementGate; + now: () => number; + isStopping: () => boolean; + cancelInferenceEnvironment: (environmentId: string) => void; + inState: (record: WorkerEnvironmentRecord, ...states: WorkerEnvironmentState[]) => boolean; + move: ( + record: WorkerEnvironmentRecord, + to: WorkerEnvironmentState, + patch?: Parameters[0]["patch"], + ) => WorkerEnvironmentRecord; + serviceError: (code: "environment_not_found" | "invalid_state", message: string) => Error; + withLock: (environmentId: string, task: () => Promise) => Promise; +}; + +export function createWorkerCredentialBroker(options: WorkerCredentialBrokerOptions) { + const { store } = options; + const tunnels = options.tunnelManager; + const now = options.now; + const inference = { cancelEnvironment: options.cancelInferenceEnvironment }; + const inState = options.inState; + const move = options.move; + const serviceError = options.serviceError; + const withLock = options.withLock; + const pendingCredentials = new Map(); + + const credentialExpiry = () => { + const ttlMs = options.workerCredentialTtlMs ?? WORKER_CREDENTIAL_TTL_MS; + if (!Number.isSafeInteger(ttlMs) || ttlMs < 1) { + throw serviceError("invalid_state", "Worker credential lifetime is invalid"); + } + const expiresAtMs = now() + ttlMs; + if (!Number.isSafeInteger(expiresAtMs)) { + throw serviceError("invalid_state", "Worker credential expiry is out of range"); + } + return expiresAtMs; + }; + + const credentialMaterial = () => createWorkerCredentialMaterial(options.generateWorkerCredential); + + const grantFrom = (params: { + credential: string; + record: ReturnType; + }): MintedWorkerCredential => { + const record = params.record; + if (!record) { + throw serviceError("invalid_state", "Worker credential persistence failed"); + } + return { + credential: params.credential, + deliveryId: record.credentialHash, + environmentId: record.environmentId, + bundleHash: record.bundleHash, + sessionId: record.sessionId, + rpcSetVersion: record.rpcSetVersion, + ownerEpoch: record.ownerEpoch, + expiresAtMs: record.expiresAtMs, + }; + }; + + const mintCredentialLocked = ( + request: WorkerCredentialBinding, + ): { credentialHash: string; grant: MintedWorkerCredential } => { + const previous = store.getCredential(request.environmentId); + if (previous) { + inference.cancelEnvironment(request.environmentId); + } + const material = credentialMaterial(); + const credential = { + environmentId: request.environmentId, + expectedOwnerEpoch: request.ownerEpoch, + credentialHash: material.credentialHash, + sessionId: request.sessionId, + rpcSetVersion: WORKER_RPC_SET_VERSION, + expiresAtMs: credentialExpiry(), + }; + const record = store.renewCredential(credential); + return { + credentialHash: material.credentialHash, + grant: grantFrom({ credential: material.credential, record }), + }; + }; + + const stageCredential = (grant: MintedWorkerCredential): MintedWorkerCredential => { + pendingCredentials.set(grant.environmentId, grant); + return grant; + }; + + const ensurePendingCredential = (record: WorkerEnvironmentRecord, sessionId: string | null) => { + const credential = store.getCredential(record.environmentId); + const pending = pendingCredentials.get(record.environmentId); + const credentialHasDurableTurn = + credential?.deliveredAtMs !== null && + credential?.ownerEpoch === record.ownerEpoch && + credential.sessionId === sessionId && + sessionId !== null && + options.placementStore?.hasWorkerTurn({ + sessionId, + environmentId: record.environmentId, + ownerEpoch: record.ownerEpoch, + }) === true; + const credentialIsCurrent = + credential?.ownerEpoch === record.ownerEpoch && + credential.sessionId === sessionId && + (credential.expiresAtMs > now() || credentialHasDurableTurn); + const pendingIsCurrent = + credentialIsCurrent && + pending?.deliveryId === credential.credentialHash && + pending.ownerEpoch === record.ownerEpoch && + pending.sessionId === sessionId; + if (credentialIsCurrent && credential.deliveredAtMs !== null) { + pendingCredentials.delete(record.environmentId); + return; + } + if (pendingIsCurrent) { + return; + } + pendingCredentials.delete(record.environmentId); + const minted = mintCredentialLocked({ + environmentId: record.environmentId, + ownerEpoch: record.ownerEpoch, + sessionId, + }); + stageCredential(minted.grant); + if (sessionId && credential?.ownerEpoch === record.ownerEpoch) { + options.liveEvents?.rotateCredential({ + credentialHash: minted.credentialHash, + environmentId: record.environmentId, + previousCredentialHash: credential.credentialHash, + runEpoch: record.ownerEpoch, + sessionId, + }); + } + }; + + const attachSession = async ( + request: WorkerCredentialBinding & { sessionId: string }, + ): Promise => { + let stopping = options.isStopping(); + if (stopping) { + throw serviceError("invalid_state", "Worker environment service is stopping"); + } + return withLock(request.environmentId, async () => { + stopping = options.isStopping(); + if (stopping) { + throw serviceError("invalid_state", "Worker environment service is stopping"); + } + const current = store.get(request.environmentId); + if (!current) { + throw serviceError( + "environment_not_found", + `Unknown worker environment: ${request.environmentId}`, + ); + } + if (current.state !== "ready" && current.state !== "idle") { + throw serviceError("invalid_state", `Cannot attach worker in state: ${current.state}`); + } + let currentBuild: WorkerInstallationArtifact; + try { + currentBuild = await options.prepareInstallation("bundle"); + } catch { + throw serviceError("invalid_state", "Current worker build identity is unavailable"); + } + if ( + !current.bootstrapReceipt || + !verifyWorkerAdmissionHandshake(current.bootstrapReceipt, currentBuild) + ) { + throw serviceError( + "invalid_state", + "Worker must bootstrap the current build before attach", + ); + } + const material = credentialMaterial(); + let attached: WorkerEnvironmentRecord; + try { + attached = store.transition({ + environmentId: request.environmentId, + from: current.state, + to: "attached", + expectedOwnerEpoch: request.ownerEpoch, + patch: { + attachedSessionIds: [request.sessionId], + credential: { + credentialHash: material.credentialHash, + sessionId: request.sessionId, + rpcSetVersion: WORKER_RPC_SET_VERSION, + expiresAtMs: credentialExpiry(), + }, + }, + }); + } catch (error) { + if (error instanceof WorkerSessionAlreadyAttachedError) { + throw serviceError("invalid_state", error.message); + } + throw error; + } + if (options.liveEvents) { + let liveSessionBound: boolean; + try { + liveSessionBound = options.liveEvents.bindSession({ + environmentId: attached.environmentId, + runEpoch: attached.ownerEpoch, + sessionId: request.sessionId, + }); + } catch { + liveSessionBound = false; + } + if (!liveSessionBound) { + move(attached, "idle"); + // Preserve the bounded attachment error after rollback fences the old worker. + await tunnels?.stop(request.environmentId, current.ownerEpoch).catch(() => undefined); + throw serviceError("invalid_state", "Attached session target is unavailable"); + } + } + pendingCredentials.delete(request.environmentId); + await tunnels?.stop(request.environmentId, current.ownerEpoch); + return stageCredential( + grantFrom({ + credential: material.credential, + record: store.getCredential(request.environmentId), + }), + ); + }); + }; + + const readPendingCredential = (binding: WorkerCredentialBinding) => { + const stopping = options.isStopping(); + if (stopping) { + return undefined; + } + const grant = pendingCredentials.get(binding.environmentId); + if ( + !grant || + grant.ownerEpoch !== binding.ownerEpoch || + grant.sessionId !== binding.sessionId + ) { + return undefined; + } + const environment = store.get(binding.environmentId); + const credential = store.getCredential(binding.environmentId); + const credentialHash = grant.deliveryId; + const checkedAtMs = now(); + if ( + !environment || + !inState(environment, "ready", "idle", "attached") || + environment.destroyRequestedAtMs !== null || + environment.ownerEpoch !== binding.ownerEpoch || + !credential || + credential.credentialHash !== credentialHash || + credential.ownerEpoch !== binding.ownerEpoch || + credential.sessionId !== binding.sessionId || + credential.deliveredAtMs !== null || + credential.expiresAtMs <= checkedAtMs + ) { + return undefined; + } + return { checkedAtMs, credentialHash, grant }; + }; + + const acquireTurnCredential = (binding: WorkerCredentialBinding & { sessionId: string }) => + withLock(binding.environmentId, async () => { + const pending = readPendingCredential(binding)?.grant; + if (pending) { + return pending; + } + const environment = store.get(binding.environmentId); + if ( + !environment || + environment.state !== "attached" || + environment.ownerEpoch !== binding.ownerEpoch || + environment.attachedSessionIds.length !== 1 || + environment.attachedSessionIds[0] !== binding.sessionId + ) { + throw serviceError("invalid_state", "Worker session credential owner is not attached"); + } + const previous = store.getCredential(binding.environmentId); + const minted = mintCredentialLocked(binding); + const grant = stageCredential(minted.grant); + if (previous?.sessionId === binding.sessionId) { + options.liveEvents?.rotateCredential({ + credentialHash: minted.credentialHash, + environmentId: binding.environmentId, + newProcessTurn: true, + previousCredentialHash: previous.credentialHash, + runEpoch: binding.ownerEpoch, + sessionId: binding.sessionId, + }); + } + return grant; + }); + + const acknowledgeCredentialDelivery = (claim: WorkerCredentialDeliveryClaim): boolean => { + const pending = readPendingCredential(claim); + if (!pending || pending.grant.deliveryId !== claim.deliveryId) { + return false; + } + store.markCredentialDelivered({ + ...claim, + credentialHash: pending.credentialHash, + deliveredAtMs: pending.checkedAtMs, + }); + pendingCredentials.delete(claim.environmentId); + return true; + }; + + return { + acknowledgeCredentialDelivery, + acquireTurnCredential, + attachSession, + clear: () => pendingCredentials.clear(), + clearEnvironment: (environmentId: string) => pendingCredentials.delete(environmentId), + credentialExpiry, + credentialMaterial, + ensurePendingCredential, + grantFrom, + readPendingCredential, + stageCredential, + takeMintedCredential: (binding: WorkerCredentialBinding) => + readPendingCredential(binding)?.grant, + }; +} + +export type WorkerCredentialBroker = ReturnType; diff --git a/src/gateway/worker-environments/environment-access.test.ts b/src/gateway/worker-environments/environment-access.test.ts new file mode 100644 index 000000000000..505380be7216 --- /dev/null +++ b/src/gateway/worker-environments/environment-access.test.ts @@ -0,0 +1,440 @@ +import { describe, expect, it, vi } from "vitest"; +import { + closeOpenClawStateDatabaseForTest, + openOpenClawStateDatabase, +} from "../../state/openclaw-state-db.js"; +import * as support from "./service.test-support.js"; +import { createWorkerEnvironmentStore } from "./store.js"; +import type { WorkerTunnelManager } from "./tunnel.js"; + +type WorkerEnvironmentServiceError = support.WorkerEnvironmentServiceError; + +describe("worker environment service", () => { + support.setupWorkerEnvironmentServiceSuite(); + + it("projects live tunnel status and fences the tunnel before provider teardown", async () => { + support.seedReady("worker-tunnel", undefined, true); + const order: string[] = []; + let tunnelStatus: "stopped" | "connected" = "stopped"; + const tunnelManager = { + status: () => tunnelStatus, + start: vi.fn(async (request) => { + tunnelStatus = "connected"; + return { + environmentId: request.environmentId, + ownerEpoch: request.ownerEpoch, + remoteSocketPath: "/tmp/worker/gateway.sock", + runWorkspaceCommand: vi.fn(), + syncWorkspace: vi.fn(), + stop: async () => {}, + }; + }), + stop: vi.fn(async () => { + tunnelStatus = "stopped"; + order.push("tunnel-stop"); + }), + stopAll: vi.fn(async () => {}), + } as unknown as WorkerTunnelManager; + const provider = support.createProvider({ + destroy: async () => { + order.push("provider-destroy"); + }, + }); + const workerService = support.createService(provider, { tunnelManager }); + + await expect( + workerService.startTunnel({ environmentId: "worker-tunnel", ownerEpoch: 0 }), + ).rejects.toThrow("owner credential is not current"); + expect(tunnelManager.start).not.toHaveBeenCalled(); + + await workerService.startTunnel({ + environmentId: "worker-tunnel", + ownerEpoch: 1, + }); + expect(tunnelManager.start).toHaveBeenCalledWith( + expect.objectContaining({ + bundleHash: support.BUNDLE_HASH, + gateway: { host: "127.0.0.1", port: 18_789 }, + sharedHost: true, + }), + ); + expect(workerService.get("worker-tunnel")).toMatchObject({ tunnelStatus: "connected" }); + + await workerService.destroy("worker-tunnel"); + expect(order).toEqual(["tunnel-stop", "provider-destroy"]); + expect(workerService.get("worker-tunnel")).toMatchObject({ + state: "destroyed", + tunnelStatus: "stopped", + }); + }); + + it("reconciles shared-host isolation for a persisted lease before tunnel startup", async () => { + support.seedReady("worker-legacy-shared"); + support.testState.stateDb.db + .prepare("UPDATE worker_environments SET shared_host = NULL WHERE environment_id = ?") + .run("worker-legacy-shared"); + closeOpenClawStateDatabaseForTest(); + support.testState.stateDb = openOpenClawStateDatabase({ + env: { OPENCLAW_STATE_DIR: support.testState.root }, + }); + support.testState.store = createWorkerEnvironmentStore({ + database: support.testState.stateDb, + now: () => support.testState.nowMs, + }); + const tunnelManager = { + status: () => "stopped" as const, + start: vi.fn(async (request: Parameters[0]) => ({ + environmentId: request.environmentId, + ownerEpoch: request.ownerEpoch, + remoteSocketPath: "/tmp/worker/gateway.sock", + runWorkspaceCommand: vi.fn(), + syncWorkspace: vi.fn(), + stop: async () => {}, + })), + stop: vi.fn(async () => {}), + stopAll: vi.fn(async () => {}), + } as unknown as WorkerTunnelManager; + let inspectionFails = true; + const provider = support.createProvider({ + inspect: async () => { + if (inspectionFails) { + throw new Error("provider unavailable"); + } + return { status: "active", sharedHost: true }; + }, + }); + const workerService = support.createService(provider, { tunnelManager }); + + expect(support.testState.store.get("worker-legacy-shared")?.sharedHost).toBeNull(); + await workerService.reconcileOnce(); + await expect( + workerService.startTunnel({ environmentId: "worker-legacy-shared", ownerEpoch: 1 }), + ).rejects.toThrow("isolation is not reconciled"); + expect(tunnelManager.start).not.toHaveBeenCalled(); + inspectionFails = false; + await workerService.reconcileOnce(); + expect(support.testState.store.get("worker-legacy-shared")?.sharedHost).toBe(true); + await workerService.startTunnel({ environmentId: "worker-legacy-shared", ownerEpoch: 1 }); + expect(tunnelManager.start).toHaveBeenCalledWith(expect.objectContaining({ sharedHost: true })); + }); + + it("fences an existing tunnel before changing its shared-host isolation", async () => { + support.seedReady("worker-isolation-change"); + const stop = vi.fn(async () => { + expect(support.testState.store.get("worker-isolation-change")?.sharedHost).toBe(false); + }); + const tunnelManager = { + status: () => "connected" as const, + start: vi.fn(), + stop, + stopAll: vi.fn(async () => {}), + } as unknown as WorkerTunnelManager; + const provider = support.createProvider({ + inspect: async () => ({ status: "active", sharedHost: true }), + }); + + await support.createService(provider, { tunnelManager }).reconcileOnce(); + + expect(stop).toHaveBeenCalledWith("worker-isolation-change"); + expect(support.testState.store.get("worker-isolation-change")?.sharedHost).toBe(true); + }); + + it("projects desktop availability only while a desktop lease is observable", () => { + const ready = support.seedReadyDesktop("worker-desktop-projection"); + const workerService = support.createService(support.createProvider()); + expect(workerService.get(ready.environmentId)).toMatchObject({ + desktopAvailable: true, + desktopApps: ["browser", "terminal"], + }); + support.testState.store.transition({ + environmentId: ready.environmentId, + from: ready.state, + to: "draining", + }); + expect(workerService.get(ready.environmentId)).toMatchObject({ + desktopAvailable: false, + desktopApps: [], + }); + }); + + it("launches only an advertised desktop app through the pinned SSH runtime", async () => { + const record = support.seedReadyDesktop("worker-desktop-launch"); + const launchApp = vi.fn(async () => {}); + const tunnelManager = { + desktop: { + acquire: vi.fn(), + attachObserver: vi.fn(), + launchApp, + stop: vi.fn(async () => {}), + stopAll: vi.fn(async () => {}), + }, + status: () => "stopped" as const, + start: vi.fn(), + stop: vi.fn(async () => {}), + stopAll: vi.fn(async () => {}), + } as unknown as WorkerTunnelManager; + const workerService = support.createService(support.createProvider(), { tunnelManager }); + + await expect( + workerService.launchDesktopApp({ environmentId: record.environmentId, app: "browser" }), + ).resolves.toEqual({ app: "browser", status: "ready" }); + expect(launchApp).toHaveBeenCalledExactlyOnceWith({ + environmentId: record.environmentId, + ownerEpoch: record.ownerEpoch, + ssh: support.SSH_ENDPOINT, + app: support.DESKTOP.apps?.[0], + resolveIdentity: expect.any(Function), + }); + }); + + it("rejects missing desktop apps and maps launcher runtime failures to typed errors", async () => { + const record = support.seedReadyDesktop("worker-desktop-launch-errors"); + const launchApp = vi.fn(async () => { + throw new Error("private SSH launcher detail"); + }); + const tunnelManager = { + desktop: { + acquire: vi.fn(), + attachObserver: vi.fn(), + launchApp, + stop: vi.fn(async () => {}), + stopAll: vi.fn(async () => {}), + }, + status: () => "stopped" as const, + start: vi.fn(), + stop: vi.fn(async () => {}), + stopAll: vi.fn(async () => {}), + } as unknown as WorkerTunnelManager; + const workerService = support.createService(support.createProvider(), { tunnelManager }); + + await expect( + workerService.launchDesktopApp({ environmentId: record.environmentId, app: "browser" }), + ).rejects.toMatchObject({ + code: "launcher_failure", + message: "worker desktop browser launcher failed; verify the app is installed and retry", + }); + support.testState.store.transition({ + environmentId: record.environmentId, + from: record.state, + to: "draining", + }); + await expect( + workerService.launchDesktopApp({ environmentId: record.environmentId, app: "terminal" }), + ).rejects.toMatchObject({ code: "invalid_state" }); + + const browserOnly = support.seedReadyDesktop("worker-desktop-browser-only", { + ...support.DESKTOP, + apps: [support.DESKTOP.apps![0]!], + }); + await expect( + workerService.launchDesktopApp({ + environmentId: browserOnly.environmentId, + app: "terminal", + }), + ).rejects.toMatchObject({ + code: "desktop_app_not_found", + message: "environment does not advertise desktop app: terminal", + }); + }); + + it("acquires a desktop tunnel and mints a one-shot websocket path", async () => { + const record = support.seedReadyDesktop("worker-desktop-observe"); + const desktopPassword = ["desktop", String.fromCharCode(45), "secret"].join(""); + const acquire = vi.fn(async () => ({ + localSocketPath: "/tmp/worker-desktop.sock", + vncPassword: desktopPassword, + })); + const tunnelManager = { + desktop: { + acquire, + attachObserver: vi.fn(), + stop: vi.fn(async () => {}), + stopAll: vi.fn(async () => {}), + }, + status: () => "stopped" as const, + start: vi.fn(), + stop: vi.fn(async () => {}), + stopAll: vi.fn(async () => {}), + } as unknown as WorkerTunnelManager; + const workerService = support.createService(support.createProvider(), { tunnelManager }); + + await expect( + workerService.observeDesktop({ environmentId: record.environmentId, control: true }), + ).resolves.toMatchObject({ + transport: "rfb", + wsPath: expect.stringMatching(/^\/worker-desktop\/observe\?token=[a-f0-9]{48}$/u), + expiresAtMs: support.testState.nowMs + 60_000, + control: true, + vncPassword: desktopPassword, + }); + expect(acquire).toHaveBeenCalledWith( + expect.objectContaining({ + environmentId: record.environmentId, + ownerEpoch: record.ownerEpoch, + desktop: support.DESKTOP, + ssh: support.SSH_ENDPOINT, + resolveIdentity: expect.any(Function), + }), + ); + }); + + it("rejects desktop observe for invalid lifecycle gates and a stopped service", async () => { + const tunnelManager = { + desktop: { + acquire: vi.fn(), + attachObserver: vi.fn(), + stop: vi.fn(async () => {}), + stopAll: vi.fn(async () => {}), + }, + status: () => "stopped" as const, + start: vi.fn(), + stop: vi.fn(async () => {}), + stopAll: vi.fn(async () => {}), + } as unknown as WorkerTunnelManager; + const workerService = support.createService(support.createProvider(), { tunnelManager }); + const requested = support.testState.store.createIntent({ + environmentId: "worker-desktop-requested", + providerId: "fake", + profileId: "development", + profileSnapshot: { settings: { region: "test" } }, + provisionOperationId: "provision:worker-desktop-requested", + }); + support.seedReady("worker-desktop-missing"); + const destroying = support.seedReadyDesktop("worker-desktop-destroy-requested"); + support.testState.store.requestDestroy({ + environmentId: destroying.environmentId, + state: destroying.state, + }); + + support.testState.config.cloudWorkers!.desktop = false; + await expect( + workerService.observeDesktop({ environmentId: requested.environmentId, control: false }), + ).rejects.toMatchObject({ + code: "invalid_state", + message: + "worker desktop observe is disabled; enable the Desktop lab in Control UI Settings -> Labs (config: cloudWorkers.desktop)", + }); + support.testState.config.cloudWorkers!.desktop = true; + + for (const environmentId of [ + requested.environmentId, + "worker-desktop-missing", + destroying.environmentId, + ]) { + await expect( + workerService.observeDesktop({ environmentId, control: false }), + ).rejects.toMatchObject({ + code: "invalid_state", + message: "environment has no desktop; desktop is a warm-time capability of the profile", + }); + } + await expect( + workerService.observeDesktop({ environmentId: "worker-desktop-unknown", control: false }), + ).rejects.toMatchObject({ code: "environment_not_found" }); + await workerService.stop(); + await expect( + workerService.observeDesktop({ environmentId: destroying.environmentId, control: false }), + ).rejects.toMatchObject({ + code: "invalid_state", + message: "Worker environment service is stopping", + }); + expect(tunnelManager.desktop.acquire).not.toHaveBeenCalled(); + }); + + it("fences a draining tunnel before reporting an unavailable provider", async () => { + support.seedReady("worker-provider-missing"); + const tunnelManager = { + status: () => "connected" as const, + start: vi.fn(), + stop: vi.fn(async () => {}), + stopAll: vi.fn(async () => {}), + } as unknown as WorkerTunnelManager; + const workerService = support.createService(support.createProvider(), { tunnelManager }); + support.testState.providersEnabled = false; + + await expect(workerService.destroy("worker-provider-missing")).rejects.toMatchObject({ + code: "provider_not_found", + } satisfies Partial); + + expect(tunnelManager.stop).toHaveBeenCalledWith("worker-provider-missing"); + expect(support.testState.store.get("worker-provider-missing")).toMatchObject({ + state: "draining", + destroyRequestedAtMs: expect.any(Number), + }); + }); + + it("does not hold the environment lock while a tunnel is connecting", async () => { + support.seedReady("worker-tunnel-pending"); + let rejectStart: ((error: Error) => void) | undefined; + const pendingStart = new Promise((_resolve, reject) => { + rejectStart = reject; + }); + const order: string[] = []; + const tunnelManager = { + status: () => "connecting" as const, + start: vi.fn(() => pendingStart), + stop: vi.fn(async () => { + order.push("tunnel-stop"); + rejectStart?.(new Error("tunnel stopped")); + }), + stopAll: vi.fn(async () => {}), + } as unknown as WorkerTunnelManager; + const provider = support.createProvider({ + destroy: async () => { + order.push("provider-destroy"); + }, + }); + const workerService = support.createService(provider, { tunnelManager }); + + const starting = workerService.startTunnel({ + environmentId: "worker-tunnel-pending", + ownerEpoch: 1, + }); + const rejectedStart = expect(starting).rejects.toThrow("tunnel stopped"); + await support.waitForFast(() => expect(tunnelManager.start).toHaveBeenCalledOnce()); + + await workerService.destroy("worker-tunnel-pending"); + + await rejectedStart; + expect(order).toEqual(["tunnel-stop", "provider-destroy"]); + }); + + it("stops a poisoned tunnel start and returns a typed deadline error", async () => { + vi.useFakeTimers(); + support.seedReady("worker-tunnel-timeout"); + let signalStarted!: () => void; + const started = new Promise((resolve) => { + signalStarted = resolve; + }); + let rejectStart!: (error: Error) => void; + const pendingStart = new Promise((_resolve, reject) => { + rejectStart = reject; + }); + const tunnelManager = { + status: () => "connecting" as const, + start: vi.fn(() => { + signalStarted(); + return pendingStart; + }), + stop: vi.fn(async () => { + rejectStart(new Error("tunnel stopped")); + }), + stopAll: vi.fn(async () => {}), + } as unknown as WorkerTunnelManager; + const workerService = support.createService(support.createProvider(), { tunnelManager }); + + const starting = workerService.startTunnel({ + environmentId: "worker-tunnel-timeout", + ownerEpoch: 1, + }); + const rejected = expect(starting).rejects.toMatchObject({ + code: "provider_failure", + message: expect.stringContaining("did not connect within 3 minutes"), + } satisfies Partial); + await started; + await vi.advanceTimersByTimeAsync(3 * 60_000); + + await rejected; + expect(tunnelManager.stop).toHaveBeenCalledWith("worker-tunnel-timeout", 1); + }); +}); diff --git a/src/gateway/worker-environments/environment-access.ts b/src/gateway/worker-environments/environment-access.ts new file mode 100644 index 000000000000..5f4a8febf4f2 --- /dev/null +++ b/src/gateway/worker-environments/environment-access.ts @@ -0,0 +1,347 @@ +import type { OpenClawConfig } from "../../config/types.js"; +import { withTimeout } from "../../infra/fs-safe.js"; +import type { WorkerProvider } from "../../plugins/types.js"; +import type { WorkerDesktopLaunchResult, WorkerDesktopObserveResult } from "./service-contract.js"; +import type { WorkerEnvironmentState } from "./state.js"; +import type { WorkerEnvironmentRecord, WorkerEnvironmentStore } from "./store.js"; +import type { WorkerTunnelRequest } from "./tunnel-contract.js"; +import type { WorkerTunnelHandle, WorkerTunnelManager } from "./tunnel.js"; +import { boundedWorkerError as boundedError } from "./worker-error.js"; + +const TUNNEL_START_TIMEOUT_MS = 3 * 60_000; + +type WorkerEnvironmentAccessOptions = { + store: WorkerEnvironmentStore; + getConfig: () => OpenClawConfig; + tunnelManager?: WorkerTunnelManager; + resolveWorkerGateway?: () => { host: "127.0.0.1" | "::1"; port: number } | undefined; + now: () => number; + identityResolverFor: ( + record: WorkerEnvironmentRecord, + provider: WorkerProvider, + leaseId: string, + ) => Parameters[0]["resolveIdentity"]; + inState: (record: WorkerEnvironmentRecord, ...states: WorkerEnvironmentState[]) => boolean; + isStopping: () => boolean; + providerFor: (providerId: string) => WorkerProvider; + serviceError: ( + code: + | "desktop_app_not_found" + | "environment_not_found" + | "invalid_state" + | "launcher_failure" + | "provider_failure" + | "unsupported_platform", + message: string, + ) => Error; + withLock: (environmentId: string, task: () => Promise) => Promise; +}; + +export function createWorkerEnvironmentAccess(options: WorkerEnvironmentAccessOptions) { + const { store } = options; + const tunnels = options.tunnelManager; + const now = options.now; + const inState = options.inState; + const providerFor = options.providerFor; + const identityResolverFor = options.identityResolverFor; + const serviceError = options.serviceError; + const withLock = options.withLock; + + const project = (record: WorkerEnvironmentRecord) => { + const desktopAvailable = + inState(record, "ready", "idle", "attached") && record.desktop !== null; + return { + ...record, + ...((record.state === "failed" || record.state === "orphaned") && record.lastError + ? { error: boundedError(record.lastError) } + : {}), + desktopAvailable, + desktopApps: desktopAvailable + ? (record.desktop?.apps?.map((app) => app.id).toSorted() ?? []) + : [], + tunnelStatus: tunnels?.status(record.environmentId) ?? ("stopped" as const), + }; + }; + + const startTunnel = async (request: WorkerTunnelRequest): Promise => { + let stopping = options.isStopping(); + if (stopping) { + throw serviceError("invalid_state", "Worker environment service is stopping"); + } + if (!tunnels) { + throw serviceError("invalid_state", "Worker tunnel runtime is unavailable"); + } + let startup: Promise | undefined; + await withLock(request.environmentId, async () => { + stopping = options.isStopping(); + if (stopping) { + throw serviceError("invalid_state", "Worker environment service is stopping"); + } + const record = store.get(request.environmentId); + if (!record) { + throw serviceError( + "environment_not_found", + `Unknown worker environment: ${request.environmentId}`, + ); + } + if ( + !inState(record, "ready", "idle", "attached") || + record.destroyRequestedAtMs !== null || + !record.leaseId || + !record.sshEndpoint || + !record.bootstrapReceipt + ) { + throw serviceError("invalid_state", `Cannot start tunnel in state: ${record.state}`); + } + if (record.sharedHost === null) { + throw serviceError( + "provider_failure", + "Worker lease isolation is not reconciled; retry after provider inspection", + ); + } + const credential = store.getCredential(request.environmentId); + if ( + !credential || + credential.ownerEpoch !== request.ownerEpoch || + credential.expiresAtMs <= now() + ) { + throw serviceError("invalid_state", "Worker tunnel owner credential is not current"); + } + const gateway = options.resolveWorkerGateway?.(); + if (!gateway) { + throw serviceError("invalid_state", "Worker gateway ingress is unavailable"); + } + const provider = providerFor(record.providerId); + // Tunnel ownership is registered synchronously by the manager. Release the durable-state + // lock while SSH connects so drain/destroy can fence an indefinitely reconnecting start. + startup = tunnels.start({ + ...request, + bundleHash: record.bootstrapReceipt.bundleHash, + gateway, + ssh: record.sshEndpoint, + sharedHost: record.sharedHost, + resolveIdentity: identityResolverFor(record, provider, record.leaseId), + }); + }); + if (!startup) { + throw serviceError("invalid_state", "Worker tunnel failed to start"); + } + const timeoutError = serviceError( + "provider_failure", + "Worker tunnel did not connect within 3 minutes; check worker SSH reachability and retry", + ); + try { + return await withTimeout(startup, TUNNEL_START_TIMEOUT_MS, { + createError: () => timeoutError, + }); + } catch (error) { + if (error !== timeoutError) { + throw error; + } + // Stop can itself block on an unkillable SSH child; detach it (rejection observed, + // entry stays manager-tracked) so the deadline error is returned on time. Epoch-fenced + // so a stale timed-out attempt can never tear down a newer owner's tunnel. + void tunnels.stop(request.environmentId, request.ownerEpoch).catch(() => undefined); + throw timeoutError; + } + }; + + const observeDesktop = async (request: { + environmentId: string; + control: boolean; + }): Promise => { + let stopping = options.isStopping(); + if (options.getConfig().cloudWorkers?.desktop !== true) { + throw serviceError( + "invalid_state", + "worker desktop observe is disabled; enable the Desktop lab in Control UI Settings -> Labs (config: cloudWorkers.desktop)", + ); + } + if (stopping) { + throw serviceError("invalid_state", "Worker environment service is stopping"); + } + if (!tunnels) { + throw serviceError("invalid_state", "Worker tunnel runtime is unavailable"); + } + let startup: Promise<{ localSocketPath: string; vncPassword?: string }> | undefined; + let ownerEpoch: number | undefined; + await withLock(request.environmentId, async () => { + stopping = options.isStopping(); + if (stopping) { + throw serviceError("invalid_state", "Worker environment service is stopping"); + } + const record = store.get(request.environmentId); + if (!record) { + throw serviceError( + "environment_not_found", + `Unknown worker environment: ${request.environmentId}`, + ); + } + if ( + !inState(record, "ready", "idle", "attached") || + record.destroyRequestedAtMs !== null || + !record.leaseId || + !record.sshEndpoint || + !record.desktop + ) { + throw serviceError( + "invalid_state", + "environment has no desktop; desktop is a warm-time capability of the profile", + ); + } + const provider = providerFor(record.providerId); + ownerEpoch = record.ownerEpoch; + startup = tunnels.desktop.acquire({ + environmentId: record.environmentId, + ownerEpoch: record.ownerEpoch, + ssh: record.sshEndpoint, + desktop: record.desktop, + resolveIdentity: identityResolverFor(record, provider, record.leaseId), + }); + }); + if (!startup || ownerEpoch === undefined) { + throw serviceError("invalid_state", "Worker desktop tunnel failed to start"); + } + const acquired = await startup; + const { WORKER_DESKTOP_OBSERVE_PATH, mintWorkerDesktopObserverToken } = + await import("./desktop-observe.js"); + const minted = mintWorkerDesktopObserverToken({ + environmentId: request.environmentId, + ownerEpoch, + control: request.control, + localSocketPath: acquired.localSocketPath, + nowMs: now(), + }); + return { + transport: "rfb", + wsPath: `${WORKER_DESKTOP_OBSERVE_PATH}?token=${minted.token}`, + expiresAtMs: minted.expiresAtMs, + control: request.control, + ...(acquired.vncPassword ? { vncPassword: acquired.vncPassword } : {}), + }; + }; + + const launchDesktopApp = async (request: { + environmentId: string; + app: "browser" | "terminal"; + }): Promise => { + let stopping = options.isStopping(); + if (options.getConfig().cloudWorkers?.desktop !== true) { + throw serviceError( + "invalid_state", + "worker desktop launch is disabled; enable the Desktop lab in Control UI Settings -> Labs (config: cloudWorkers.desktop)", + ); + } + if (stopping) { + throw serviceError("invalid_state", "Worker environment service is stopping"); + } + if (!tunnels) { + throw serviceError("invalid_state", "Worker tunnel runtime is unavailable"); + } + const requireLaunchable = () => { + stopping = options.isStopping(); + if (stopping) { + throw serviceError("invalid_state", "Worker environment service is stopping"); + } + const record = store.get(request.environmentId); + if (!record) { + throw serviceError( + "environment_not_found", + `Unknown worker environment: ${request.environmentId}`, + ); + } + if ( + !inState(record, "ready", "idle", "attached") || + record.destroyRequestedAtMs !== null || + !record.leaseId || + !record.sshEndpoint || + !record.desktop + ) { + throw serviceError( + "invalid_state", + "environment has no desktop; desktop is a warm-time capability of the profile", + ); + } + const app = record.desktop.apps?.find((candidate) => candidate.id === request.app); + if (!app) { + throw serviceError( + "desktop_app_not_found", + `environment does not advertise desktop app: ${request.app}`, + ); + } + return { app, record }; + }; + + let startup: Promise | undefined; + let launchEpoch: number | undefined; + await withLock(request.environmentId, async () => { + const { app, record } = requireLaunchable(); + const provider = providerFor(record.providerId); + launchEpoch = record.ownerEpoch; + startup = tunnels.desktop.launchApp({ + environmentId: record.environmentId, + ownerEpoch: record.ownerEpoch, + ssh: record.sshEndpoint, + app, + resolveIdentity: identityResolverFor(record, provider, record.leaseId), + }); + }); + if (!startup || launchEpoch === undefined) { + throw serviceError("launcher_failure", "Worker desktop app launcher failed to start"); + } + try { + await startup; + } catch (error) { + if ( + error && + typeof error === "object" && + "code" in error && + error.code === "unsupported_platform" + ) { + throw serviceError( + "unsupported_platform", + "desktop app launch is not supported on Windows gateway hosts", + ); + } + // A teardown aborts the SSH child before mutating the durable row. Wait for the + // environment lock, then report the authoritative lifecycle state instead of a launch error. + await withLock(request.environmentId, async () => { + const { record } = requireLaunchable(); + if (record.ownerEpoch !== launchEpoch) { + throw serviceError("invalid_state", "Worker desktop app launch owner changed"); + } + }); + throw serviceError( + "launcher_failure", + `worker desktop ${request.app} launcher failed; verify the app is installed and retry`, + ); + } + await withLock(request.environmentId, async () => { + const { record } = requireLaunchable(); + if (record.ownerEpoch !== launchEpoch) { + throw serviceError("invalid_state", "Worker desktop app launch owner changed"); + } + }); + return { app: request.app, status: "ready" }; + }; + + const stopTunnel = async (environmentId: string, ownerEpoch?: number): Promise => { + await withLock(environmentId, async () => { + await tunnels?.stop(environmentId, ownerEpoch); + }); + }; + + return { + get: (environmentId: string) => { + const record = store.get(environmentId); + return record ? project(record) : undefined; + }, + launchDesktopApp, + list: () => store.list().map(project), + observeDesktop, + project, + startTunnel, + stopAllTunnels: () => tunnels?.stopAll() ?? Promise.resolve(), + stopTunnel, + }; +} diff --git a/src/gateway/worker-environments/placement-dispatch-test-fixtures.ts b/src/gateway/worker-environments/placement-dispatch-test-fixtures.ts index a54864afa36c..2fbef98e3279 100644 --- a/src/gateway/worker-environments/placement-dispatch-test-fixtures.ts +++ b/src/gateway/worker-environments/placement-dispatch-test-fixtures.ts @@ -9,7 +9,7 @@ import { createWorkerSessionPlacementStore, type WorkerSessionPlacementRecord, } from "./placement-store.js"; -import { workerEnvironmentIdForIdempotencyKey } from "./service.js"; +import { deriveEnvironmentIntent } from "./service-contract.js"; type WorkerDispatchRequest = Parameters< ReturnType["dispatch"] @@ -90,9 +90,9 @@ export function seedActivePlacement( } export function createDispatchEnvironmentFixtures(generation = 1) { - const environmentId = workerEnvironmentIdForIdempotencyKey( + const environmentId = deriveEnvironmentIntent( `session-dispatch:${REQUEST.sessionId}:${generation}`, - ); + ).environmentId; const profileSnapshot: WorkerProfile = { settings: { region: "test" } }; const bootstrapReceipt: WorkerAdmissionHandshake = { bundleHash: BUNDLE_HASH, diff --git a/src/gateway/worker-environments/placement-dispatch.test.ts b/src/gateway/worker-environments/placement-dispatch.test.ts index dd94ad72fdc1..01977af63639 100644 --- a/src/gateway/worker-environments/placement-dispatch.test.ts +++ b/src/gateway/worker-environments/placement-dispatch.test.ts @@ -17,7 +17,7 @@ import { } from "./placement-dispatch-test-fixtures.js"; import { createHarness } from "./placement-dispatch-test-harness.js"; import { createWorkerSessionPlacementStore } from "./placement-store.js"; -import { workerEnvironmentIdForIdempotencyKey } from "./service.js"; +import { deriveEnvironmentIntent } from "./service-contract.js"; describe("worker placement dispatch", () => { let root: string; @@ -566,9 +566,9 @@ describe("worker placement dispatch", () => { to: "provisioning", expectedGeneration: interrupted.generation, patch: { - environmentId: workerEnvironmentIdForIdempotencyKey( + environmentId: deriveEnvironmentIntent( `session-dispatch:${REQUEST.sessionId}:${interrupted.generation}`, - ), + ).environmentId, }, }); } diff --git a/src/gateway/worker-environments/placement-dispatch.ts b/src/gateway/worker-environments/placement-dispatch.ts index b75c48c2d2c9..ad5206924205 100644 --- a/src/gateway/worker-environments/placement-dispatch.ts +++ b/src/gateway/worker-environments/placement-dispatch.ts @@ -15,7 +15,8 @@ import type { WorkerPlacementDispatchRequest, WorkerPlacementReclaimRequest, } from "./service-contract.js"; -import { type WorkerEnvironmentService, workerEnvironmentIdForIdempotencyKey } from "./service.js"; +import { deriveEnvironmentIntent } from "./service-contract.js"; +import type { WorkerEnvironmentService } from "./service.js"; import { WorkerTunnelOwnerDisconnectedError } from "./tunnel-contract.js"; import type { WorkerWorkspaceResultConflict } from "./workspace-conflicts.js"; import { @@ -156,7 +157,7 @@ export function createWorkerPlacementDispatchService(options: WorkerPlacementDis }); const localPath = await options.resolveWorkspacePath(request); const idempotencyKey = `session-dispatch:${request.sessionId}:${placement.generation}`; - const expectedEnvironmentId = workerEnvironmentIdForIdempotencyKey(idempotencyKey); + const expectedEnvironmentId = deriveEnvironmentIntent(idempotencyKey).environmentId; placement = placements.transition({ sessionId: request.sessionId, from: "requested", diff --git a/src/gateway/worker-environments/placement-worker-gate.ts b/src/gateway/worker-environments/placement-worker-gate.ts index e1dfc66fa5ac..9600af63d517 100644 --- a/src/gateway/worker-environments/placement-worker-gate.ts +++ b/src/gateway/worker-environments/placement-worker-gate.ts @@ -3,11 +3,30 @@ import type { WorkerSessionPlacementStore, WorkerSessionTurnClaim, } from "./placement-store.js"; -import type { - WorkerPlacementBinding, - WorkerPlacementTurnBinding, - WorkerSessionPlacementGate, -} from "./service.js"; + +type WorkerPlacementBinding = Readonly<{ + sessionId: string; + environmentId: string; + ownerEpoch: number; +}>; + +export type WorkerPlacementTurnBinding = WorkerPlacementBinding & + Readonly<{ + runId: string; + }>; + +export type WorkerSessionPlacementGate = { + hasWorkerTurn(binding: WorkerPlacementBinding): boolean; + validateWorkerTurn(binding: WorkerPlacementTurnBinding): boolean; + isWorkerTurnToolAuthorized(binding: WorkerPlacementTurnBinding, toolName: string): boolean; + updateAckCursors( + binding: WorkerPlacementTurnBinding & { + transcriptSeq?: number; + liveSeq?: number; + workspaceResultPending?: boolean; + }, + ): void; +}; function claimForBinding( record: WorkerSessionPlacementRecord | undefined, diff --git a/src/gateway/worker-environments/provider-lifecycle.ts b/src/gateway/worker-environments/provider-lifecycle.ts new file mode 100644 index 000000000000..47d1b22d0121 --- /dev/null +++ b/src/gateway/worker-environments/provider-lifecycle.ts @@ -0,0 +1,661 @@ +import { isDeepStrictEqual } from "node:util"; +import { expectDefined } from "@openclaw/normalization-core"; +import { MAX_TIMER_TIMEOUT_MS } from "@openclaw/normalization-core/number-coercion"; +import { + type WorkerAdmissionHandshake, + WORKER_RPC_SET_VERSION, +} from "../../../packages/gateway-protocol/src/schema/worker-admission.js"; +import type { OpenClawConfig } from "../../config/types.js"; +import type { SecretRef } from "../../config/types.secrets.js"; +import { validateCloudWorkerProfileSettings } from "../../config/zod-schema.cloud-workers.js"; +import { normalizeCapabilityProviderId } from "../../plugins/provider-registry-shared.js"; +import { + WorkerProviderError, + type WorkerLease, + type WorkerProfile, + type WorkerProvider, + type WorkerSshEndpoint, + type WorkerSshIdentity, +} from "../../plugins/types.js"; +import { verifyWorkerAdmissionHandshake } from "./admission.js"; +import type { WorkerInstallationArtifact } from "./bundle.js"; +import type { WorkerCredentialBroker } from "./credential-broker.js"; +import { deriveEnvironmentIntent } from "./service-contract.js"; +import { requireWorkerLease, requireWorkerLeaseStatus } from "./service-validation.js"; +import type { WorkerEnvironmentState } from "./state.js"; +import type { + WorkerEnvironmentRecord, + WorkerEnvironmentStore, + WorkerEnvironmentTransitionPatch as TransitionPatch, +} from "./store.js"; +import type { WorkerTunnelManager } from "./tunnel.js"; +import { boundedWorkerError as boundedError } from "./worker-error.js"; + +const ORPHANED_LEASE_ERROR = "Worker provider no longer recognizes the lease"; + +type WorkerProviderLifecycleOptions = { + store: WorkerEnvironmentStore; + getConfig: () => OpenClawConfig; + resolveProvider: (providerId: string) => WorkerProvider | undefined; + prepareInstallation: ( + install: WorkerInstallationArtifact["install"], + ) => Promise; + bootstrapWorker: (params: { + operationId: string; + sshEndpoint: WorkerSshEndpoint; + installation: WorkerInstallationArtifact; + resolveIdentity: (keyRef: SecretRef) => Promise; + signal: AbortSignal; + }) => Promise; + resolveSshIdentity?: (params: { + provider: WorkerProvider; + leaseId: string; + profile: WorkerProfile; + keyRef: SecretRef; + }) => Promise; + providerCallTimeoutMs?: number; + tunnelManager?: WorkerTunnelManager; + credentialBroker: WorkerCredentialBroker; + callBootstrap: (run: (signal: AbortSignal) => Promise) => Promise; + callProvider: (environmentId: string, run: () => Promise, timeoutMs?: number) => Promise; + inState: (record: WorkerEnvironmentRecord, ...states: WorkerEnvironmentState[]) => boolean; + isServiceError: (error: unknown, code: string) => boolean; + isStopping: () => boolean; + move: ( + record: WorkerEnvironmentRecord, + to: WorkerEnvironmentState, + patch?: TransitionPatch, + ) => WorkerEnvironmentRecord; + saveError: (record: WorkerEnvironmentRecord, error: unknown) => WorkerEnvironmentRecord; + serviceError: ( + code: + | "bootstrap_failure" + | "environment_not_found" + | "invalid_profile" + | "invalid_state" + | "profile_not_found" + | "provider_failure" + | "provider_not_found", + message: string, + ) => Error; + withLock: (environmentId: string, task: () => Promise) => Promise; +}; + +function requireProviderProvisionTimeoutMs(timeoutMs: number | undefined): number | undefined { + if (timeoutMs === undefined) { + return undefined; + } + if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > MAX_TIMER_TIMEOUT_MS) { + throw new WorkerProviderError( + `Worker provider provision timeout must be an integer from 1 through ${MAX_TIMER_TIMEOUT_MS}ms`, + ); + } + return timeoutMs; +} + +export function createWorkerProviderLifecycle(options: WorkerProviderLifecycleOptions) { + const { store } = options; + const tunnels = options.tunnelManager; + const callBootstrap = options.callBootstrap; + const callProvider = options.callProvider; + const inState = options.inState; + const move = options.move; + const saveError = options.saveError; + const serviceError = options.serviceError; + const withLock = options.withLock; + const { + credentialExpiry, + credentialMaterial, + ensurePendingCredential, + grantFrom, + stageCredential, + } = options.credentialBroker; + + function requireWorkerProfile(value: unknown): WorkerProfile { + const error = validateCloudWorkerProfileSettings(value); + if (error) { + throw serviceError("invalid_profile", error); + } + return value as WorkerProfile; + } + + const lifecycleLease = (record: WorkerEnvironmentRecord, leaseId: string) => ({ + leaseId, + profile: requireWorkerProfile(record.profileSnapshot.settings), + }); + + const identityResolverFor = ( + record: WorkerEnvironmentRecord, + provider: WorkerProvider, + leaseId: string, + ) => { + const profile = requireWorkerProfile(record.profileSnapshot.settings); + const resolveSshIdentity = options.resolveSshIdentity; + return async (keyRef: SecretRef) => { + if (!resolveSshIdentity) { + throw new Error("Worker SSH identity resolution is unavailable"); + } + return await callProvider(record.environmentId, () => + resolveSshIdentity({ provider, leaseId, profile, keyRef }), + ); + }; + }; + + const providerFor = (providerId: string): WorkerProvider => { + const provider = options.resolveProvider(providerId); + if (provider) { + return provider; + } + throw serviceError("provider_not_found", `Worker provider is unavailable: ${providerId}`); + }; + + const installFor = (record: WorkerEnvironmentRecord): WorkerInstallationArtifact["install"] => { + const install = record.profileSnapshot.install; + if (install === undefined || install === "bundle") { + return "bundle"; + } + if (install === "npm") { + return "npm"; + } + throw serviceError("invalid_profile", "Worker profile has an invalid install method"); + }; + + const prepareInstallation = (record: WorkerEnvironmentRecord) => + options.prepareInstallation(installFor(record)); + + const finishProvenDestroy = (record: WorkerEnvironmentRecord) => { + const destroying = beginDestroy(record); + if (destroying.teardownTerminalState !== "failed") { + return move(destroying, "destroyed"); + } + return move(destroying, "failed", { + leaseId: null, + sshEndpoint: null, + sharedHost: false, + lastError: destroying.lastError ?? "Worker bootstrap failed after provider teardown", + }); + }; + + const failBootstrap = async ( + record: WorkerEnvironmentRecord, + leaseId: string, + provider: WorkerProvider, + error: unknown, + ): Promise => { + const detail = boundedError(error); + const requested = store.requestDestroy({ + environmentId: record.environmentId, + state: record.state, + terminalState: "failed", + lastError: detail, + }); + const draining = move(requested, "draining", { lastError: detail }); + await tunnels?.stop(record.environmentId); + const destroying = move(draining, "destroying", { lastError: detail }); + try { + await callProvider(record.environmentId, () => + provider.destroy(lifecycleLease(record, leaseId)), + ); + } catch (cleanupError) { + // An indeterminate destroy must remain retryable; never hide a possibly-live paid lease + // behind terminal failed state. + saveError( + destroying, + new Error(`${detail}; provider teardown pending: ${boundedError(cleanupError)}`), + ); + throw serviceError( + "bootstrap_failure", + `Worker bootstrap failed; teardown is pending: ${detail}`, + ); + } + finishProvenDestroy(destroying); + throw serviceError("bootstrap_failure", `Worker bootstrap failed: ${detail}`); + }; + + const finishBootstrap = async ( + record: WorkerEnvironmentRecord, + provider: WorkerProvider, + installation: WorkerInstallationArtifact, + ) => { + if (record.state !== "bootstrapping" || !record.leaseId || !record.sshEndpoint) { + throw serviceError("invalid_state", "Worker bootstrap requires a provisioned SSH lease"); + } + let receipt: WorkerAdmissionHandshake; + try { + receipt = await callBootstrap((signal) => + options.bootstrapWorker({ + operationId: record.provisionOperationId, + sshEndpoint: record.sshEndpoint, + installation, + resolveIdentity: identityResolverFor(record, provider, record.leaseId), + signal, + }), + ); + if (!verifyWorkerAdmissionHandshake(receipt, installation)) { + throw new Error("Worker bootstrap receipt does not match the expected build identity"); + } + } catch (error) { + return await failBootstrap(record, record.leaseId, provider, error); + } + const material = credentialMaterial(); + // Receipt, owner epoch, and credential hash commit together. A failed write leaves the + // durable lease bootstrapping so reconcile can retry without admitting a partial identity. + const ready = move(record, "ready", { + bootstrapReceipt: receipt, + credential: { + credentialHash: material.credentialHash, + sessionId: null, + rpcSetVersion: WORKER_RPC_SET_VERSION, + expiresAtMs: credentialExpiry(), + }, + }); + const grant = grantFrom({ + credential: material.credential, + record: store.getCredential(record.environmentId), + }); + stageCredential(grant); + return ready; + }; + + const finishProvision = async ( + record: WorkerEnvironmentRecord, + provider: WorkerProvider, + preparedInstallation?: WorkerInstallationArtifact, + ) => { + let lease: WorkerLease; + try { + const profile = requireWorkerProfile(record.profileSnapshot.settings); + const providerTimeoutMs = + options.providerCallTimeoutMs === undefined + ? requireProviderProvisionTimeoutMs(provider.resolveProvisionTimeoutMs?.(profile)) + : undefined; + lease = requireWorkerLease( + await callProvider( + record.environmentId, + () => provider.provision(profile, record.provisionOperationId), + providerTimeoutMs, + ), + ); + } catch (error) { + const detail = boundedError(error); + if ( + error instanceof WorkerProviderError || + options.isServiceError(error, "invalid_profile") + ) { + move(record, "failed", { lastError: detail }); + throw serviceError("invalid_profile", `Worker provider rejected profile: ${detail}`); + } + saveError(record, error); + throw serviceError("provider_failure", `Worker provider operation failed: ${detail}`); + } + // A timeout can happen after allocation; retain the same operation id for safe replay. + const patch = { + leaseId: lease.leaseId, + sshEndpoint: lease.ssh, + sharedHost: lease.sharedHost === true, + desktop: lease.desktop ?? null, + }; + const bootstrapping = move(record, "bootstrapping", patch); + if (record.destroyRequestedAtMs !== null) { + return bootstrapping; + } + let installation = preparedInstallation; + if (!installation) { + try { + // A persisted provisioning row can represent an allocation whose response was lost. + // Replay the idempotent provider operation before packaging can terminalize that lease. + installation = await prepareInstallation(bootstrapping); + } catch (error) { + return await failBootstrap(bootstrapping, lease.leaseId, provider, error); + } + } + return finishBootstrap(bootstrapping, provider, installation); + }; + + const resumeProvision = async ( + record: WorkerEnvironmentRecord, + provider = providerFor(record.providerId), + ) => { + let installation: WorkerInstallationArtifact | undefined; + if (record.state === "requested" && record.destroyRequestedAtMs === null) { + try { + // Fresh requests package before allocation. Once provisioning is durable, provider replay + // must happen first because the previous response may have been lost after allocation. + installation = await prepareInstallation(record); + } catch (error) { + const detail = boundedError(error); + move(record, "failed", { lastError: detail }); + throw serviceError( + "bootstrap_failure", + `Worker installation preparation failed: ${detail}`, + ); + } + } + const provisioning = record.state === "requested" ? move(record, "provisioning") : record; + return finishProvision(provisioning, provider, installation); + }; + + const cancelRequested = (record: WorkerEnvironmentRecord) => + move(record, "failed", { lastError: "Provisioning canceled before provider allocation" }); + + const beginDrain = (record: WorkerEnvironmentRecord) => { + const failurePatch = + record.teardownTerminalState === "failed" ? { lastError: record.lastError } : undefined; + return inState(record, "bootstrapping", "ready", "attached", "idle") + ? move(record, "draining", failurePatch) + : record; + }; + + const beginDestroy = (record: WorkerEnvironmentRecord) => { + const failurePatch = + record.teardownTerminalState === "failed" ? { lastError: record.lastError } : undefined; + const draining = beginDrain(record); + if (draining.state === "draining") { + return move(draining, "destroying", failurePatch); + } + if (draining.state === "destroying") { + return draining; + } + throw serviceError("invalid_state", `Cannot destroy worker in state: ${record.state}`); + }; + + const finishDestroy = async (r: WorkerEnvironmentRecord, provider?: WorkerProvider) => { + if (!r.leaseId) { + throw serviceError("invalid_state", "Worker environment has no lease"); + } + const leaseId = r.leaseId; + const draining = beginDrain(r); + await tunnels?.stop(r.environmentId); + const owningProvider = provider ?? providerFor(r.providerId); + const destroying = beginDestroy(draining); + try { + await callProvider(r.environmentId, () => owningProvider.destroy(lifecycleLease(r, leaseId))); + } catch (error) { + saveError(destroying, error); + throw serviceError("provider_failure", "Worker provider operation failed"); + } + return finishProvenDestroy(destroying); + }; + + const reconcileRecord = async (initialRecord: WorkerEnvironmentRecord): Promise => { + let record = initialRecord; + if (record.state === "requested" && record.destroyRequestedAtMs !== null) { + return void cancelRequested(record); + } + let currentBundle: WorkerInstallationArtifact | undefined; + if (record.destroyRequestedAtMs === null && inState(record, "ready", "idle", "attached")) { + try { + currentBundle = await options.prepareInstallation("bundle"); + if ( + record.bootstrapReceipt && + verifyWorkerAdmissionHandshake(record.bootstrapReceipt, currentBundle) + ) { + const sessionId = record.state === "attached" ? record.attachedSessionIds[0] : null; + if (record.state !== "attached" || sessionId) { + ensurePendingCredential(record, sessionId ?? null); + record = store.get(record.environmentId) ?? record; + } + } + } catch { + // Provider inspection and the state-specific path below retain their existing retry policy. + } + } + let provider: WorkerProvider; + try { + provider = providerFor(record.providerId); + } catch (error) { + saveError(record, error); + return; + } + const leaseId = record.leaseId; + if (!leaseId) { + const provisioned = await resumeProvision(record, provider).catch(() => undefined); + if (provisioned?.state === "bootstrapping") { + await finishDestroy(provisioned, provider).catch(() => undefined); + } + return; + } + const inspection = await callProvider(record.environmentId, () => + provider.inspect(lifecycleLease(record, leaseId)), + ) + .then(requireWorkerLeaseStatus) + .catch((error: unknown) => { + saveError(record, error); + return undefined; + }); + if (!inspection) { + return; + } + const { status } = inspection; + const teardownExpected = record.destroyRequestedAtMs !== null || record.state === "destroying"; + if (status === "destroyed" || (status === "unknown" && teardownExpected)) { + const requested = + record.destroyRequestedAtMs === null + ? store.requestDestroy({ + environmentId: record.environmentId, + state: record.state, + ...(status === "destroyed" && !teardownExpected + ? { + terminalState: "failed", + lastError: "Worker environment disappeared before teardown was requested", + } + : {}), + }) + : record; + const draining = beginDrain(requested); + await tunnels?.stop(record.environmentId); + finishProvenDestroy(draining); + return; + } + if (status === "unknown") { + const draining = + record.state === "draining" + ? record + : move(record, "draining", { lastError: ORPHANED_LEASE_ERROR }); + await tunnels?.stop(record.environmentId); + move(draining, "orphaned", { lastError: ORPHANED_LEASE_ERROR }); + return; + } + const inspectedSharedHost = inspection.sharedHost === true; + if (record.sharedHost !== null && record.sharedHost !== inspectedSharedHost) { + // Workspace actions capture isolation at tunnel creation. Fence the old actions before + // committing a provider-owned change so no reconciliation can use stale host scope. + await tunnels?.stop(record.environmentId); + } + record = store.reconcileSharedHost({ + environmentId: record.environmentId, + state: record.state, + leaseId, + sharedHost: inspectedSharedHost, + }); + if (record.destroyRequestedAtMs !== null) { + await finishDestroy(record, provider).catch(() => undefined); + return; + } + if (record.state === "attached") { + if ( + currentBundle && + (!record.bootstrapReceipt || + !verifyWorkerAdmissionHandshake(record.bootstrapReceipt, currentBundle)) + ) { + // A new Gateway build rejects the old worker at admission. This is expected lifecycle + // teardown, not a bootstrap failure. `leaseId` above came from this record, so provider + // inspection and destruction share the same durable lease identity. + await finishDestroy(record, provider).catch(() => undefined); + } + return; + } + if (record.state === "draining" && record.destroyRequestedAtMs === null) { + // Draining without destroy intent is durable provider-loss cleanup. + await tunnels?.stop(record.environmentId); + move(record, "orphaned", { lastError: record.lastError ?? ORPHANED_LEASE_ERROR }); + return; + } + if (inState(record, "bootstrapping", "ready", "idle")) { + let installation = currentBundle; + try { + // Bundle identity is local and canonical for both install channels. A matching admitted + // receipt must not depend on npm registry availability during routine reconciliation. + installation ??= await options.prepareInstallation("bundle"); + } catch (error) { + if (record.bootstrapReceipt && inState(record, "ready", "idle")) { + saveError(record, error); + return; + } + await failBootstrap(record, leaseId, provider, error).catch(() => undefined); + return; + } + if ( + record.bootstrapReceipt && + verifyWorkerAdmissionHandshake(record.bootstrapReceipt, installation) + ) { + ensurePendingCredential(record, null); + return; + } + if (installFor(record) === "npm") { + try { + installation = await options.prepareInstallation("npm"); + } catch (error) { + await failBootstrap(record, leaseId, provider, error).catch(() => undefined); + return; + } + } + const bootstrapping = + record.state === "bootstrapping" ? record : move(record, "bootstrapping"); + await tunnels?.stop(record.environmentId, record.ownerEpoch); + await finishBootstrap(bootstrapping, provider, installation).catch(() => undefined); + return; + } + if (inState(record, "draining", "destroying")) { + await finishDestroy(record, provider).catch(() => undefined); + } + }; + + const createWithProfile = async ( + profileId: string, + idempotencyKey: string, + inherited?: { + providerId: string; + profileSnapshot: WorkerProfile; + }, + ) => { + let stopping = options.isStopping(); + if (stopping) { + throw serviceError("invalid_state", "Worker environment service is stopping"); + } + const normalizedProfileId = profileId.trim(); + if (!normalizedProfileId || normalizedProfileId !== profileId) { + throw serviceError("invalid_profile", "Worker profile id must be non-empty and trimmed"); + } + const { environmentId, provisionOperationId } = deriveEnvironmentIntent(idempotencyKey); + return withLock(environmentId, async () => { + stopping = options.isStopping(); + if (stopping) { + throw serviceError("invalid_state", "Worker environment service is stopping"); + } + const existing = store.get(environmentId); + if (existing) { + if ( + existing.profileId !== normalizedProfileId || + (inherited !== undefined && + (existing.providerId !== inherited.providerId || + !isDeepStrictEqual(existing.profileSnapshot, inherited.profileSnapshot))) + ) { + throw serviceError("invalid_profile", "Idempotency key belongs to another profile"); + } + if (existing.destroyRequestedAtMs !== null) { + return existing; + } + if (!existing.leaseId && inState(existing, "requested", "provisioning")) { + return resumeProvision(existing); + } + return existing; + } + let provider: WorkerProvider; + let providerId: string; + let profileSnapshot: WorkerProfile; + if (inherited) { + providerId = normalizeCapabilityProviderId(inherited.providerId) ?? inherited.providerId; + if (providerId !== inherited.providerId) { + throw serviceError("invalid_profile", "Inherited worker provider id is not canonical"); + } + provider = providerFor(providerId); + const resolvedProviderId = normalizeCapabilityProviderId(provider.id) ?? provider.id; + if (resolvedProviderId !== providerId) { + throw serviceError("invalid_profile", "Inherited worker provider identity changed"); + } + profileSnapshot = requireWorkerProfile(inherited.profileSnapshot); + } else { + const profiles = options.getConfig().cloudWorkers?.profiles; + if (!profiles || !Object.hasOwn(profiles, normalizedProfileId)) { + throw serviceError("profile_not_found", `Unknown worker profile: ${normalizedProfileId}`); + } + const profile = expectDefined( + profiles[normalizedProfileId], + "profiles entry at normalized profile id", + ); + provider = providerFor(profile.provider); + providerId = normalizeCapabilityProviderId(provider.id) ?? provider.id; + const settings = requireWorkerProfile(profile.settings ?? {}); + profileSnapshot = requireWorkerProfile({ + install: profile.install ?? "bundle", + settings, + }); + } + const intent = store.createIntent({ + environmentId, + providerId, + profileId: normalizedProfileId, + profileSnapshot, + provisionOperationId, + }); + return resumeProvision(intent, provider); + }); + }; + + const destroy = async ( + environmentId: string, + destroyOptions: { requireUnattached?: boolean } = {}, + ) => { + const stopping = options.isStopping(); + if (stopping) { + throw serviceError("invalid_state", "Worker environment service is stopping"); + } + return withLock(environmentId, async () => { + let record = store.get(environmentId); + if (!record) { + throw serviceError("environment_not_found", `Unknown worker environment: ${environmentId}`); + } + if (inState(record, "destroyed", "failed", "orphaned")) { + return record; + } + if (destroyOptions.requireUnattached && record.attachedSessionIds.length > 0) { + throw serviceError( + "invalid_state", + "Attached cloud workers must be stopped through sessions.reclaim", + ); + } + record = store.requestDestroy({ environmentId, state: record.state }); + if (record.state === "requested") { + return cancelRequested(record); + } + if (record.leaseId) { + record = beginDrain(record); + } + if (!record.leaseId) { + const provider = providerFor(record.providerId); + record = await resumeProvision(record, provider); + return finishDestroy(record, provider); + } + return finishDestroy(record); + }); + }; + + return { + createWithProfile, + destroy, + identityResolverFor, + providerFor, + reconcileRecord, + }; +} diff --git a/src/gateway/worker-environments/provider-provisioning.test.ts b/src/gateway/worker-environments/provider-provisioning.test.ts new file mode 100644 index 000000000000..64623d563170 --- /dev/null +++ b/src/gateway/worker-environments/provider-provisioning.test.ts @@ -0,0 +1,868 @@ +import path from "node:path"; +import { expectDefined } from "@openclaw/normalization-core"; +import { MAX_TIMER_TIMEOUT_MS } from "@openclaw/normalization-core/number-coercion"; +import { describe, expect, it, vi } from "vitest"; +import { WorkerProviderError, type WorkerProfile } from "../../plugins/types.js"; +import { + closeOpenClawStateDatabaseForTest, + openOpenClawStateDatabase, +} from "../../state/openclaw-state-db.js"; +import type { GatewaySessionRow } from "../session-utils.types.js"; +import { writeSessionStore } from "../test-helpers.js"; +import { directSessionReq } from "../test/server-sessions.test-helpers.js"; +import { hashWorkerCredential } from "./credential.js"; +import { createWorkerPlacementDispatchService } from "./placement-dispatch.js"; +import { createWorkerSessionPlacementStore } from "./placement-store.js"; +import * as support from "./service.test-support.js"; +import { createWorkerEnvironmentStore } from "./store.js"; +import { createWorkerWorkspaceOperationCoordinator } from "./workspace-operation-coordinator.js"; + +type WorkerEnvironmentServiceError = support.WorkerEnvironmentServiceError; + +describe("worker environment service", () => { + support.setupWorkerEnvironmentServiceSuite(); + + it("persists intent and an immutable profile snapshot before provisioning", async () => { + const operationIds: string[] = []; + const provider = support.createProvider({ + provision: async (profile, operationId) => { + operationIds.push(operationId); + expect(support.testState.store.list()[0]).toMatchObject({ + state: "provisioning", + provisionOperationId: operationId, + profileSnapshot: { + install: "bundle", + settings: { region: "test" }, + }, + }); + support.getDevelopmentProfile().settings = { region: "mutated" }; + expect(profile).toEqual({ region: "test" }); + return { leaseId: "lease-1", ssh: support.SSH_ENDPOINT }; + }, + }); + + const workerService = support.createService(provider); + const result = await workerService.create("development", "request-1"); + const repeated = await workerService.create("development", "request-1"); + + expect(result).toMatchObject({ state: "ready", leaseId: "lease-1", ownerEpoch: 1 }); + expect(repeated.environmentId).toBe(result.environmentId); + expect(operationIds).toHaveLength(1); + expect(operationIds[0]).toMatch(/^provision:v2:[a-f0-9]{64}$/u); + expect(result.profileSnapshot).toMatchObject({ settings: { region: "test" } }); + expect(support.testState.store.getCredential(result.environmentId)).toMatchObject({ + credentialHash: hashWorkerCredential(support.CREDENTIAL), + ownerEpoch: 1, + sessionId: null, + }); + const persistedCredential = support.testState.stateDb.db + .prepare("SELECT * FROM worker_environment_credentials WHERE environment_id = ?") + .get(result.environmentId); + expect(persistedCredential).toMatchObject({ + credential_hash: hashWorkerCredential(support.CREDENTIAL), + }); + expect(JSON.stringify(persistedCredential)).not.toContain(support.CREDENTIAL); + const binding = { environmentId: result.environmentId, ownerEpoch: 1, sessionId: null }; + const grant = workerService.takeMintedCredential(binding); + expect(grant).toMatchObject({ + credential: support.CREDENTIAL, + ownerEpoch: 1, + sessionId: null, + }); + expect(workerService.acknowledgeCredentialDelivery(grant!)).toBe(true); + expect(support.testState.store.getCredential(result.environmentId)).toMatchObject({ + deliveredAtMs: support.testState.nowMs, + }); + expect(workerService.takeMintedCredential(binding)).toBeUndefined(); + }); + + it("creates a nested environment from its parent's snapshot after config drift", async () => { + const provisionedProfiles: WorkerProfile[] = []; + let lease = 0; + let credential = 0; + const workerService = support.createService( + support.createProvider({ + provision: async (profile) => { + provisionedProfiles.push(structuredClone(profile)); + lease += 1; + return { leaseId: `lease-${lease}`, ssh: support.SSH_ENDPOINT }; + }, + }), + { + generateWorkerCredential: () => `nested-worker-credential-${(credential += 1)}`, + }, + ); + const parent = await workerService.create("development", "parent-profile-snapshot"); + support.getDevelopmentProfile().settings = { region: "mutated" }; + + const child = await workerService.createFromProfileSnapshot( + { + profileId: parent.profileId, + providerId: parent.providerId, + profileSnapshot: parent.profileSnapshot, + }, + "child-profile-snapshot", + ); + + expect(provisionedProfiles).toEqual([{ region: "test" }, { region: "test" }]); + expect(child).toMatchObject({ + profileId: parent.profileId, + providerId: parent.providerId, + profileSnapshot: parent.profileSnapshot, + }); + }); + + it("stays bootstrapping until the SSH install receipt is durable", async () => { + let finishBootstrap: (() => void) | undefined; + const bootstrapPending = new Promise((resolve) => { + finishBootstrap = resolve; + }); + support.testState.bootstrapWorker = vi.fn(async () => { + await bootstrapPending; + return support.BOOTSTRAP_RECEIPT; + }); + const creation = support + .createService(support.createProvider()) + .create("development", "request-bootstrap"); + + await support.waitForFast(() => + expect(support.testState.store.list()[0]).toMatchObject({ + state: "bootstrapping", + bootstrapReceipt: null, + }), + ); + finishBootstrap?.(); + + await expect(creation).resolves.toMatchObject({ + state: "ready", + bootstrapReceipt: support.BOOTSTRAP_RECEIPT, + }); + }); + + it("records installation preparation failure before allocating a lease", async () => { + support.testState.prepareInstallation = vi.fn(async () => { + throw new Error("npm install requires a released gateway package"); + }); + const provision = vi.fn(support.createProvider().provision); + const workerService = support.createService(support.createProvider({ provision })); + + await expect( + workerService.create("development", "request-preparation-failure"), + ).rejects.toMatchObject({ + code: "bootstrap_failure", + message: expect.stringContaining("npm install requires a released gateway package"), + } satisfies Partial); + + expect(provision).not.toHaveBeenCalled(); + expect(support.testState.store.list()[0]).toMatchObject({ + state: "failed", + leaseId: null, + lastError: "npm install requires a released gateway package", + }); + expect(workerService.list()[0]).toMatchObject({ + state: "failed", + error: "npm install requires a released gateway package", + }); + }); + + it("keeps a remotely bootstrapped lease retryable when receipt persistence fails", async () => { + const durableStore = support.testState.store; + let persistenceFails = true; + support.testState.store = { + ...support.testState.store, + transition(input) { + if (persistenceFails && input.from === "bootstrapping" && input.to === "ready") { + persistenceFails = false; + throw new Error("receipt database write failed"); + } + return durableStore.transition(input); + }, + }; + const destroy = vi.fn(async () => {}); + const workerService = support.createService(support.createProvider({ destroy })); + + await expect( + workerService.create("development", "request-receipt-write-failure"), + ).rejects.toThrow("receipt database write failed"); + expect(support.testState.store.list()[0]).toMatchObject({ + state: "bootstrapping", + leaseId: "lease-1", + }); + expect(destroy).not.toHaveBeenCalled(); + + await workerService.reconcileOnce(); + expect(support.testState.store.list()[0]).toMatchObject({ + state: "ready", + bootstrapReceipt: support.BOOTSTRAP_RECEIPT, + }); + expect(support.testState.bootstrapWorker).toHaveBeenCalledTimes(2); + }); + + it("tears down the lease and records a bounded bootstrap failure", async () => { + // Assembled at runtime so review-bundle secret scanners do not flag a key-shaped literal. + const secret = [ + String.fromCharCode(115, 107), + "proj", + "bootstrap", + "abcdefghijklmnopqrstuvwxyz", + ].join("-"); + support.testState.bootstrapWorker = vi.fn(async () => { + throw new Error(`remote bootstrap rejected ${secret}`); + }); + const destroy = vi.fn(async () => {}); + const workerService = support.createService(support.createProvider({ destroy })); + + const creation = workerService.create("development", "request-bootstrap-failure"); + await expect(creation).rejects.toMatchObject({ + code: "bootstrap_failure", + message: expect.stringContaining("Worker bootstrap failed: remote bootstrap rejected"), + } satisfies Partial); + await expect(creation).rejects.not.toThrow(secret); + + expect(destroy).toHaveBeenCalledTimes(1); + expect(support.testState.store.list()[0]).toMatchObject({ + state: "failed", + leaseId: null, + sshEndpoint: null, + bootstrapReceipt: null, + lastError: expect.stringContaining("remote bootstrap rejected"), + }); + expect(support.testState.store.list()[0]?.lastError).not.toContain(secret); + }); + + it("projects bounded bootstrap detail through sessions.describe after failed dispatch", async () => { + // Assembled at runtime so review-bundle secret scanners do not flag a key-shaped literal. + const secret = [ + String.fromCharCode(115, 107), + "proj", + "placement", + "abcdefghijklmnopqrstuvwxyz", + ].join("-"); + support.testState.bootstrapWorker = vi.fn(async () => { + throw new Error(`remote bootstrap rejected ${secret} ${"failure ".repeat(200)}`); + }); + const workerService = support.createService(support.createProvider()); + const placements = createWorkerSessionPlacementStore({ + database: support.testState.stateDb, + now: () => support.testState.nowMs, + }); + const dispatch = createWorkerPlacementDispatchService({ + placements, + environments: workerService, + workspaceOperations: createWorkerWorkspaceOperationCoordinator(), + runLocalBarrier: async ({ startDispatch }) => startDispatch(), + runActivationBarrier: async ({ activate }) => activate(), + runReclaimBarrier: async ({ reclaim }) => await reclaim("/gateway/workspace"), + resolveWorkspacePath: async () => "/gateway/workspace", + reportWorkspaceResultConflict: async () => {}, + resolveWorkspaceResultConflict: async () => undefined, + }); + + await expect( + dispatch.dispatch({ + sessionId: "session-bootstrap-failure", + sessionKey: "agent:main:session-bootstrap-failure", + agentId: "main", + profileId: "development", + }), + ).rejects.toThrow("Worker bootstrap failed: remote bootstrap rejected"); + + const persisted = expectDefined( + placements.get("session-bootstrap-failure"), + "failed worker placement", + ); + const sessionStorePath = path.join(support.testState.root, "sessions.json"); + await writeSessionStore({ + entries: { main: { sessionId: persisted.sessionId, updatedAt: support.testState.nowMs } }, + storePath: sessionStorePath, + }); + const described = await directSessionReq<{ session: GatewaySessionRow | null }>( + "sessions.describe", + { key: "main" }, + { + context: { + getRuntimeConfig: () => ({ session: { store: sessionStorePath } }), + workerSessionPlacementService: placements, + }, + }, + ); + const describedPlacement = described.payload?.session?.placement; + expect(described).toMatchObject({ ok: true }); + expect(describedPlacement).toMatchObject({ + state: "failed", + recoveryError: expect.stringContaining("remote bootstrap rejected"), + }); + if (describedPlacement?.state !== "failed") { + throw new Error("sessions.describe did not project the failed worker placement"); + } + expect(describedPlacement.recoveryError).not.toContain(secret); + expect(describedPlacement.recoveryError.length).toBeLessThanOrEqual(1_024); + }); + + it("keeps an indeterminate bootstrap teardown retryable", async () => { + support.testState.bootstrapWorker = vi.fn(async () => { + throw new Error("remote bootstrap failed"); + }); + let teardownFails = true; + const workerService = support.createService( + support.createProvider({ + destroy: async () => { + if (teardownFails) { + throw new Error("provider teardown timed out"); + } + }, + }), + ); + + await expect( + workerService.create("development", "request-bootstrap-cleanup"), + ).rejects.toMatchObject({ + code: "bootstrap_failure", + message: "Worker bootstrap failed; teardown is pending: remote bootstrap failed", + } satisfies Partial); + expect(support.testState.store.list()[0]).toMatchObject({ + state: "destroying", + leaseId: "lease-1", + destroyRequestedAtMs: expect.any(Number), + teardownTerminalState: "failed", + lastError: "remote bootstrap failed", + }); + + teardownFails = false; + await workerService.reconcileOnce(); + expect(support.testState.store.list()[0]).toMatchObject({ + state: "failed", + leaseId: null, + sshEndpoint: null, + lastError: expect.stringContaining("remote bootstrap failed"), + }); + }); + + it("bounds worker identity resolution as a provider operation", async () => { + const events: string[] = []; + let finishIdentity: (() => void) | undefined; + const identityPending = new Promise((resolve) => { + finishIdentity = resolve; + }); + support.testState.bootstrapWorker = vi.fn(async ({ installation, resolveIdentity, signal }) => { + signal.addEventListener("abort", () => void events.push("abort"), { once: true }); + await resolveIdentity(support.SSH_ENDPOINT.keyRef); + return { + bundleHash: installation.bundleHash, + openclawVersion: installation.openclawVersion, + protocolFeatures: [...installation.protocolFeatures], + }; + }); + const destroy = vi.fn(async () => { + events.push("destroy"); + }); + const workerService = support.createService(support.createProvider({ destroy }), { + providerCallTimeoutMs: 5, + resolveSshIdentity: async () => { + events.push("identity:start"); + await identityPending; + events.push("identity:end"); + return { kind: "path", path: "/keys/worker" }; + }, + }); + + const creation = workerService.create("development", "request-identity-timeout"); + const creationResult = expect(creation).rejects.toMatchObject({ + code: "bootstrap_failure", + } satisfies Partial); + try { + await support.waitForFast(() => + expect(support.testState.store.list()[0]).toMatchObject({ state: "destroying" }), + ); + expect(events).toEqual(["identity:start", "abort"]); + expect(destroy).not.toHaveBeenCalled(); + } finally { + finishIdentity?.(); + } + + await creationResult; + expect(destroy).toHaveBeenCalledOnce(); + expect(events).toEqual(["identity:start", "abort", "identity:end", "destroy"]); + expect(support.testState.store.list()[0]).toMatchObject({ state: "failed", leaseId: null }); + }); + + it("aborts a timed-out SSH bootstrap before tearing down its lease", async () => { + const events: string[] = []; + support.testState.bootstrapWorker = vi.fn( + async ({ signal }) => + await new Promise((_resolve, reject) => { + signal.addEventListener( + "abort", + () => { + events.push("abort"); + reject(new Error("SSH bootstrap aborted")); + }, + { once: true }, + ); + }), + ); + const destroy = vi.fn(async () => { + events.push("destroy"); + }); + const workerService = support.createService(support.createProvider({ destroy }), { + bootstrapCallTimeoutMs: 10, + }); + + await expect( + workerService.create("development", "request-bootstrap-timeout"), + ).rejects.toMatchObject({ + code: "bootstrap_failure", + } satisfies Partial); + + expect(events).toEqual(["abort", "destroy"]); + expect(support.testState.store.list()[0]).toMatchObject({ state: "failed", leaseId: null }); + }); + + it("adopts one committed provision across a service and store restart", async () => { + const physicalLeases = new Set(); + const operationIds: string[] = []; + const destroyed: string[] = []; + let creates = 0; + let loseFirstReply = true; + const provider = () => + support.createProvider({ + provision: async (_profile, operationId) => { + operationIds.push(operationId); + if (!physicalLeases.has("lease-restarted")) { + creates += 1; + physicalLeases.add("lease-restarted"); + } + if (loseFirstReply) { + loseFirstReply = false; + throw new Error("provider response was lost after commit"); + } + return { leaseId: "lease-restarted", ssh: support.SSH_ENDPOINT }; + }, + destroy: async ({ leaseId }) => { + destroyed.push(leaseId); + physicalLeases.delete(leaseId); + }, + }); + const first = support.createService(provider()); + + await expect(first.create("development", "request-restart-replay")).rejects.toMatchObject({ + code: "provider_failure", + } satisfies Partial); + const environmentId = expectDefined( + support.testState.store.list()[0], + "persisted provision intent", + ).environmentId; + const operationId = expectDefined( + support.testState.store.get(environmentId), + "persisted provision record", + ).provisionOperationId; + expect(operationId).toMatch(/^provision:v2:[a-f0-9]{64}$/u); + expect(support.testState.store.get(environmentId)).toMatchObject({ + state: "provisioning", + leaseId: null, + }); + + await first.stop(); + support.testState.service = undefined; + closeOpenClawStateDatabaseForTest(); + support.testState.stateDb = openOpenClawStateDatabase({ + env: { OPENCLAW_STATE_DIR: support.testState.root }, + }); + support.testState.store = createWorkerEnvironmentStore({ + database: support.testState.stateDb, + now: () => support.testState.nowMs, + }); + + const restarted = support.createService(provider()); + restarted.start(); + await support.waitForFast(() => + expect(support.testState.store.get(environmentId)).toMatchObject({ + state: "ready", + leaseId: "lease-restarted", + lastError: null, + }), + ); + await restarted.destroy(environmentId); + + expect(creates).toBe(1); + expect(operationIds).toEqual([operationId, operationId]); + expect(destroyed).toEqual(["lease-restarted"]); + expect(physicalLeases.size).toBe(0); + expect(support.testState.store.get(environmentId)).toMatchObject({ + state: "destroyed", + leaseId: "lease-restarted", + }); + }); + + it("records a permanent legacy provision replay failure without allocating", async () => { + const legacyOperationId = `provision:${"0".repeat(64)}`; + const intent = support.testState.store.createIntent({ + environmentId: "worker-legacy-provision", + providerId: "fake", + profileId: "development", + profileSnapshot: { settings: { region: "test" } }, + provisionOperationId: legacyOperationId, + }); + support.testState.store.transition({ + environmentId: intent.environmentId, + from: intent.state, + to: "provisioning", + }); + const allocate = vi.fn(async () => ({ leaseId: "must-not-exist", ssh: support.SSH_ENDPOINT })); + const provider = support.createProvider({ + provision: async (_profile, operationId) => { + if (operationId === legacyOperationId) { + throw new WorkerProviderError("Legacy Crabbox provision state cannot be replayed safely"); + } + return await allocate(); + }, + }); + + await support.createService(provider).reconcileOnce(); + + expect(allocate).not.toHaveBeenCalled(); + expect(support.testState.store.get(intent.environmentId)).toMatchObject({ + state: "failed", + leaseId: null, + lastError: "Legacy Crabbox provision state cannot be replayed safely", + }); + }); + + it("does not resolve a provider provision timeout when the service override is set", async () => { + const resolveProvisionTimeoutMs = vi.fn(() => { + throw new Error("provider timeout hook must not run"); + }); + const workerService = support.createService( + support.createProvider({ resolveProvisionTimeoutMs }), + { + providerCallTimeoutMs: 1_000, + }, + ); + + await expect( + workerService.create("development", "request-provider-timeout-override"), + ).resolves.toMatchObject({ state: "ready" }); + expect(resolveProvisionTimeoutMs).not.toHaveBeenCalled(); + }); + + it.each([ + ["zero", 0], + ["negative", -1], + ["fractional", 1.5], + ["non-finite", Number.NaN], + ["timer overflow", MAX_TIMER_TIMEOUT_MS + 1], + ])("rejects a %s provider provision timeout before allocation", async (_label, timeoutMs) => { + const provision = vi.fn(async () => ({ + leaseId: "lease-invalid-timeout", + ssh: support.SSH_ENDPOINT, + })); + const workerService = support.createService( + support.createProvider({ + provision, + resolveProvisionTimeoutMs: () => timeoutMs, + }), + ); + + await expect( + workerService.create("development", `request-invalid-provider-timeout-${String(timeoutMs)}`), + ).rejects.toMatchObject({ + code: "invalid_profile", + message: expect.stringContaining("Worker provider provision timeout must be an integer"), + } satisfies Partial); + expect(provision).not.toHaveBeenCalled(); + }); + + it("serializes destroy and provision replay behind a timed-out provider operation", async () => { + const events: string[] = []; + const operationIds: string[] = []; + let active = 0; + let maxActive = 0; + let originalProvisionCalls = 0; + let finishFirstProvision: (() => void) | undefined; + const firstProvisionPending = new Promise((resolve) => { + finishFirstProvision = resolve; + }); + const destroy = vi.fn(async () => { + events.push("destroy:start"); + active += 1; + maxActive = Math.max(maxActive, active); + active -= 1; + events.push("destroy:end"); + }); + const provider = support.createProvider({ + provision: async (_profile, operationId) => { + originalProvisionCalls += 1; + const call = originalProvisionCalls; + operationIds.push(operationId); + events.push(`provision:${call}:start`); + active += 1; + maxActive = Math.max(maxActive, active); + if (call === 1) { + await firstProvisionPending; + } + active -= 1; + events.push(`provision:${call}:end`); + return { leaseId: "lease-timeout-replay", ssh: support.SSH_ENDPOINT }; + }, + destroy, + resolveProvisionTimeoutMs: () => 20, + }); + const workerService = support.createService(provider); + const creation = workerService.create("development", "request-provider-timeout-race"); + const creationResult = expect(creation).rejects.toMatchObject({ + code: "provider_failure", + } satisfies Partial); + let environmentId: string | undefined; + let teardownResult: Promise | undefined; + try { + await support.waitForFast(() => expect(events).toEqual(["provision:1:start"])); + const queuedEnvironmentId = expectDefined( + support.testState.store.list()[0], + "timed-out provision row", + ).environmentId; + environmentId = queuedEnvironmentId; + const teardown = workerService.destroy(queuedEnvironmentId); + teardownResult = expect(teardown).resolves.toMatchObject({ state: "destroyed" }); + await creationResult; + await support.waitForFast(() => + expect( + support.testState.store.get(queuedEnvironmentId)?.destroyRequestedAtMs, + ).not.toBeNull(), + ); + expect(originalProvisionCalls).toBe(1); + expect(destroy).not.toHaveBeenCalled(); + expect(maxActive).toBe(1); + } finally { + finishFirstProvision?.(); + } + + await teardownResult; + const finalEnvironmentId = expectDefined(environmentId, "timed-out provision environment id"); + expect(operationIds).toHaveLength(2); + expect(new Set(operationIds).size).toBe(1); + expect(maxActive).toBe(1); + expect(events).toEqual([ + "provision:1:start", + "provision:1:end", + "provision:2:start", + "provision:2:end", + "destroy:start", + "destroy:end", + ]); + expect(support.testState.store.get(finalEnvironmentId)).toMatchObject({ state: "destroyed" }); + }); + + it("adopts an indeterminate allocation before a replay preparation failure", async () => { + const events: string[] = []; + let preparationFails = false; + support.testState.prepareInstallation = vi.fn(async () => { + events.push("prepare"); + if (preparationFails) { + throw new Error("persisted bundle is unavailable"); + } + return support.BUNDLE_ARTIFACT; + }); + let provisionCalls = 0; + const operationIds: string[] = []; + const provider = support.createProvider({ + provision: async (_profile, operationId) => { + events.push("provision"); + provisionCalls += 1; + operationIds.push(operationId); + if (provisionCalls === 1) { + throw new Error("provision response was lost"); + } + return { leaseId: "lease-replayed", ssh: support.SSH_ENDPOINT }; + }, + destroy: async () => void events.push("destroy"), + }); + const workerService = support.createService(provider); + + await expect( + workerService.create("development", "request-lost-provision"), + ).rejects.toMatchObject({ + code: "provider_failure", + } satisfies Partial); + preparationFails = true; + await workerService.reconcileOnce(); + + expect(events).toEqual(["prepare", "provision", "provision", "prepare", "destroy"]); + expect(new Set(operationIds).size).toBe(1); + expect(support.testState.store.list()[0]).toMatchObject({ + state: "failed", + leaseId: null, + sshEndpoint: null, + teardownTerminalState: "failed", + lastError: "persisted bundle is unavailable", + }); + }); + + it.each([ + ["missing result", null, "invalid provision result"], + [ + "malformed SSH endpoint", + { leaseId: "lease-invalid", ssh: { ...support.SSH_ENDPOINT, keyRef: "not-a-secret-ref" } }, + "SSH key must be a canonical SecretRef", + ], + [ + "excessive SSH fallback ports", + { + leaseId: "lease-invalid", + ssh: { + ...support.SSH_ENDPOINT, + fallbackPorts: Array.from({ length: 11 }, (_, index) => 2300 + index), + }, + }, + "SSH fallback ports cannot exceed 10", + ], + [ + "invalid shared-host declaration", + { leaseId: "lease-invalid", ssh: support.SSH_ENDPOINT, sharedHost: "yes" }, + "invalid provision result", + ], + [ + "unsupported desktop protocol", + { + leaseId: "lease-invalid", + ssh: support.SSH_ENDPOINT, + desktop: { protocol: "rdp", port: 5900 }, + }, + 'desktop protocol must be "rfb"', + ], + [ + "invalid desktop port", + { + leaseId: "lease-invalid", + ssh: support.SSH_ENDPOINT, + desktop: { protocol: "rfb", port: 0 }, + }, + "desktop port must be an integer", + ], + [ + "relative desktop password path", + { + leaseId: "lease-invalid", + ssh: support.SSH_ENDPOINT, + desktop: { protocol: "rfb", port: 5900, passwordFilePath: "vnc.password" }, + }, + "desktop password file path must be absolute", + ], + [ + "unrecognized desktop app metadata", + { + leaseId: "lease-invalid", + ssh: support.SSH_ENDPOINT, + desktop: { + protocol: "rfb", + port: 5900, + apps: [ + { + id: "browser", + executablePath: "/usr/local/bin/openclaw-worker-browser", + cdpPort: 9222, + command: "chromium", + }, + ], + }, + }, + "browser desktop app contains unknown fields", + ], + ])("keeps %s from a provider retryable", async (_name, result, error) => { + const workerService = support.createService( + support.createProvider({ provision: async () => result as never }), + ); + + await expect(workerService.create("development", "request-malformed")).rejects.toMatchObject({ + code: "provider_failure", + message: expect.stringContaining(error), + } satisfies Partial); + expect(support.testState.store.list()[0]).toMatchObject({ + state: "provisioning", + lastError: expect.stringContaining(error), + }); + }); + + it("rejects plaintext secret fields before persisting intent", async () => { + support.getDevelopmentProfile().settings = { + keyRef: "not-a-secret-ref", + }; + const provision = vi.fn(support.createProvider().provision); + + await expect( + support + .createService(support.createProvider({ provision })) + .create("development", "request-secret"), + ).rejects.toMatchObject({ code: "invalid_profile" }); + expect(provision).not.toHaveBeenCalled(); + expect(support.testState.store.list()).toEqual([]); + }); + + it("records permanent provider profile rejection as terminal", async () => { + let provisionCalls = 0; + const provider = support.createProvider({ + provision: async () => { + provisionCalls += 1; + throw new WorkerProviderError("region is required"); + }, + }); + const workerService = support.createService(provider); + + await expect(workerService.create("development", "request-invalid")).rejects.toMatchObject({ + code: "invalid_profile", + message: expect.stringContaining("region is required"), + } satisfies Partial); + const record = expectDefined( + support.testState.store.list()[0], + "store.list()[0] test invariant", + ); + expect(record).toMatchObject({ state: "failed", lastError: "region is required" }); + + await workerService.reconcileOnce(); + await expect(workerService.destroy(record.environmentId)).resolves.toMatchObject({ + state: "failed", + }); + expect(provisionCalls).toBe(1); + }); + + it("rejects non-canonical profile ids before persistence", async () => { + const workerService = support.createService(support.createProvider()); + + await expect(workerService.create(" development ", "request-spaced")).rejects.toMatchObject({ + code: "invalid_profile", + } satisfies Partial); + expect(support.testState.store.list()).toEqual([]); + }); + + it.each(["direct destroy", "restart reconcile"] as const)( + "cancels a requested intent without allocating on %s", + async (mode) => { + const intent = support.testState.store.createIntent({ + environmentId: `worker-cancel-${mode}`, + providerId: "fake", + profileId: "development", + profileSnapshot: { settings: { region: "test" } }, + provisionOperationId: `provision:cancel-${mode}`, + }); + const provision = vi.fn(support.createProvider().provision); + const workerService = support.createService(support.createProvider({ provision })); + + if (mode === "direct destroy") { + await workerService.destroy(intent.environmentId); + } else { + support.testState.store.requestDestroy({ + environmentId: intent.environmentId, + state: "requested", + }); + support.testState.providersEnabled = false; + await workerService.reconcileOnce(); + } + + expect(provision).not.toHaveBeenCalled(); + expect(support.testState.store.get(intent.environmentId)).toMatchObject({ + state: "failed", + lastError: "Provisioning canceled before provider allocation", + destroyRequestedAtMs: expect.any(Number), + }); + }, + ); +}); diff --git a/src/gateway/worker-environments/provider-reconciliation.test.ts b/src/gateway/worker-environments/provider-reconciliation.test.ts new file mode 100644 index 000000000000..f546edbea6ab --- /dev/null +++ b/src/gateway/worker-environments/provider-reconciliation.test.ts @@ -0,0 +1,613 @@ +import { describe, expect, it, vi } from "vitest"; +import * as support from "./service.test-support.js"; +import type { WorkerTunnelManager } from "./tunnel.js"; + +type WorkerEnvironmentServiceError = support.WorkerEnvironmentServiceError; +type WorkerLifecycleLease = support.WorkerLifecycleLease; + +describe("worker environment service", () => { + support.setupWorkerEnvironmentServiceSuite(); + + it("adopts a matching milestone-1 row that predates worker credentials", async () => { + const environmentId = "worker-milestone-one"; + support.seedReady(environmentId); + support.testState.stateDb.db + .prepare("DELETE FROM worker_environment_credentials WHERE environment_id = ?") + .run(environmentId); + support.testState.stateDb.db + .prepare("UPDATE worker_environments SET owner_epoch = 0 WHERE environment_id = ?") + .run(environmentId); + const workerService = support.createService( + support.createProvider({ + inspect: async () => { + throw new Error("provider unavailable"); + }, + }), + ); + + await workerService.reconcileOnce(); + + expect(support.testState.store.get(environmentId)?.ownerEpoch).toBe(1); + expect(support.testState.store.getCredential(environmentId)).toMatchObject({ + ownerEpoch: 1, + sessionId: null, + }); + expect( + workerService.takeMintedCredential({ environmentId, ownerEpoch: 1, sessionId: null }), + ).toMatchObject({ + credential: support.CREDENTIAL, + ownerEpoch: 1, + }); + expect(support.testState.store.get(environmentId)?.lastError).toBe("provider unavailable"); + expect(support.testState.bootstrapWorker).not.toHaveBeenCalled(); + }); + + it("inspects a persisted lease with its profile snapshot after profile removal", async () => { + support.seedBootstrapping("worker-crash"); + support.testState.config.cloudWorkers!.profiles = {}; + const inspected: WorkerLifecycleLease[] = []; + const provider = support.createProvider({ + inspect: async (lease) => { + inspected.push(lease); + return { status: "active" }; + }, + provision: async () => { + throw new Error("provision must not run for a known lease"); + }, + }); + + await support.createService(provider).reconcileOnce(); + + expect(inspected).toEqual([{ leaseId: "lease:worker-crash", profile: { region: "test" } }]); + expect(support.testState.store.get("worker-crash")).toMatchObject({ + state: "ready", + bootstrapReceipt: support.BOOTSTRAP_RECEIPT, + }); + expect(support.testState.prepareInstallation).toHaveBeenCalledWith("bundle"); + expect(support.testState.bootstrapWorker).toHaveBeenCalledTimes(1); + }); + + it("skips an active lease whose durable receipt matches the lifecycle bundle", async () => { + support.seedReady("worker-current"); + + await support.createService(support.createProvider()).reconcileOnce(); + + expect(support.testState.store.get("worker-current")).toMatchObject({ + state: "ready", + bootstrapReceipt: support.BOOTSTRAP_RECEIPT, + }); + expect(support.testState.prepareInstallation).toHaveBeenCalledWith("bundle"); + expect(support.testState.bootstrapWorker).not.toHaveBeenCalled(); + }); + + it("re-enters bootstrapping when the durable receipt has a stale bundle hash", async () => { + const bootstrapping = support.seedBootstrapping("worker-stale"); + support.testState.store.transition({ + environmentId: bootstrapping.environmentId, + from: "bootstrapping", + to: "ready", + patch: support.readyPatch(bootstrapping.environmentId, { + ...support.BOOTSTRAP_RECEIPT, + bundleHash: "b".repeat(64), + }), + }); + + await support.createService(support.createProvider()).reconcileOnce(); + + expect(support.testState.store.get("worker-stale")).toMatchObject({ + state: "ready", + bootstrapReceipt: support.BOOTSTRAP_RECEIPT, + }); + expect(support.testState.bootstrapWorker).toHaveBeenCalledTimes(1); + }); + + it("tears down an attached worker whose admitted bundle is stale", async () => { + const environmentId = "worker-attached-stale"; + support.seedBootstrapping(environmentId); + const ready = support.testState.store.transition({ + environmentId, + from: "bootstrapping", + to: "ready", + patch: support.readyPatch(environmentId, { + ...support.BOOTSTRAP_RECEIPT, + bundleHash: "b".repeat(64), + }), + }); + support.testState.store.transition({ + environmentId, + from: ready.state, + to: "attached", + patch: support.attachedPatch(environmentId, "session-1"), + }); + const destroy = vi.fn(async () => {}); + + await support.createService(support.createProvider({ destroy })).reconcileOnce(); + + expect(destroy).toHaveBeenCalledOnce(); + expect(destroy).toHaveBeenCalledWith({ + leaseId: `lease:${environmentId}`, + profile: { region: "test" }, + }); + expect(support.testState.store.get(environmentId)).toMatchObject({ + state: "destroyed", + leaseId: `lease:${environmentId}`, + attachedSessionIds: [], + lastError: null, + }); + }); + + it("does not resolve npm while an admitted receipt matches the local bundle", async () => { + const environmentId = "worker-current-npm"; + support.seedReady(environmentId, "npm"); + support.testState.prepareInstallation = vi.fn(async (install) => { + if (install === "bundle") { + return support.BUNDLE_ARTIFACT; + } + throw new Error("npm registry is unavailable"); + }); + const destroy = vi.fn(async () => {}); + const workerService = support.createService(support.createProvider({ destroy })); + + await workerService.reconcileOnce(); + + expect(support.testState.prepareInstallation).toHaveBeenCalledTimes(1); + expect(support.testState.prepareInstallation).toHaveBeenCalledWith("bundle"); + expect(destroy).not.toHaveBeenCalled(); + expect(support.testState.store.get(environmentId)).toMatchObject({ + state: "ready", + leaseId: `lease:${environmentId}`, + bootstrapReceipt: support.BOOTSTRAP_RECEIPT, + lastError: null, + }); + }); + + it("keeps an admitted lease retryable when local bundle identity is unavailable", async () => { + const environmentId = "worker-current-bundle-unavailable"; + support.seedReady(environmentId, "npm"); + const attachedId = "worker-attached-bundle-unavailable"; + support.seedReady(attachedId); + support.testState.store.transition({ + environmentId: attachedId, + from: "ready", + to: "attached", + patch: support.attachedPatch(attachedId, "session-1"), + }); + support.testState.stateDb.db + .prepare("DELETE FROM worker_environment_credentials WHERE environment_id = ?") + .run(attachedId); + support.testState.prepareInstallation = vi.fn(async () => { + throw new Error("local bundle identity is unavailable"); + }); + const destroy = vi.fn(async () => {}); + const workerService = support.createService(support.createProvider({ destroy })); + + await workerService.reconcileOnce(); + + expect(destroy).not.toHaveBeenCalled(); + expect(support.testState.store.get(environmentId)).toMatchObject({ + state: "ready", + leaseId: `lease:${environmentId}`, + bootstrapReceipt: support.BOOTSTRAP_RECEIPT, + lastError: "local bundle identity is unavailable", + }); + expect(support.testState.store.getCredential(attachedId)).toBeUndefined(); + expect( + workerService.takeMintedCredential({ + environmentId: attachedId, + ownerEpoch: 2, + sessionId: "session-1", + }), + ).toBeUndefined(); + }); + + it.each(["bootstrapping", "ready", "idle"] as const)( + "tears down a persisted %s lease when mismatched npm preparation fails", + async (state) => { + const environmentId = `worker-prepare-${state}`; + const bootstrapping = support.seedBootstrapping(environmentId, "npm"); + if (state !== "bootstrapping") { + const ready = support.testState.store.transition({ + environmentId, + from: bootstrapping.state, + to: "ready", + patch: support.readyPatch(environmentId, { + ...support.BOOTSTRAP_RECEIPT, + bundleHash: "c".repeat(64), + }), + }); + if (state === "idle") { + support.testState.store.transition({ environmentId, from: ready.state, to: "idle" }); + } + } + support.testState.prepareInstallation = vi.fn(async (install) => { + if (install === "bundle") { + return support.BUNDLE_ARTIFACT; + } + throw new Error("released npm artifact is unavailable"); + }); + const order: string[] = []; + const tunnelManager = { + status: () => "connected" as const, + start: vi.fn(), + stop: vi.fn(async () => { + order.push("tunnel-stop"); + }), + stopAll: vi.fn(async () => {}), + } as unknown as WorkerTunnelManager; + const destroy = vi.fn(async () => { + order.push("provider-destroy"); + }); + + await support + .createService(support.createProvider({ destroy }), { tunnelManager }) + .reconcileOnce(); + + expect(order).toEqual(["tunnel-stop", "provider-destroy"]); + expect(destroy).toHaveBeenCalledWith({ + leaseId: `lease:${environmentId}`, + profile: { region: "test" }, + }); + expect(support.testState.store.get(environmentId)).toMatchObject({ + state: "failed", + leaseId: null, + sshEndpoint: null, + teardownTerminalState: "failed", + lastError: "released npm artifact is unavailable", + }); + expect(support.testState.prepareInstallation).toHaveBeenCalledWith("bundle"); + expect(support.testState.prepareInstallation).toHaveBeenCalledWith("npm"); + expect(support.testState.bootstrapWorker).not.toHaveBeenCalled(); + }, + ); + + it("retries indeterminate teardown after a reconcile preparation failure and restart", async () => { + const environmentId = "worker-prepare-teardown-retry"; + const bootstrapping = support.seedBootstrapping(environmentId, "npm"); + support.testState.store.transition({ + environmentId, + from: bootstrapping.state, + to: "ready", + patch: support.readyPatch(environmentId, { + ...support.BOOTSTRAP_RECEIPT, + bundleHash: "c".repeat(64), + }), + }); + support.testState.prepareInstallation = vi.fn(async (install) => { + if (install === "bundle") { + return support.BUNDLE_ARTIFACT; + } + throw new Error("released npm artifact is unavailable"); + }); + let teardownFails = true; + const destroy = vi.fn(async () => { + if (teardownFails) { + throw new Error("provider teardown timed out"); + } + }); + const provider = support.createProvider({ destroy }); + const workerService = support.createService(provider); + + await workerService.reconcileOnce(); + expect(support.testState.store.get(environmentId)).toMatchObject({ + state: "destroying", + leaseId: `lease:${environmentId}`, + teardownTerminalState: "failed", + lastError: "released npm artifact is unavailable", + }); + + await workerService.stop(); + teardownFails = false; + await support.createService(provider).reconcileOnce(); + + expect(destroy).toHaveBeenCalledTimes(2); + expect(support.testState.store.get(environmentId)).toMatchObject({ + state: "failed", + leaseId: null, + sshEndpoint: null, + teardownTerminalState: "failed", + lastError: "released npm artifact is unavailable", + }); + }); + + it("uses the snapshotted npm selection after live config changes", async () => { + support.getDevelopmentProfile().install = "npm"; + const provider = support.createProvider({ + provision: async () => { + support.getDevelopmentProfile().install = "bundle"; + return { leaseId: "lease-npm", ssh: support.SSH_ENDPOINT }; + }, + }); + + const result = await support.createService(provider).create("development", "request-npm"); + + expect(result).toMatchObject({ + state: "ready", + profileSnapshot: { install: "npm" }, + }); + expect(support.testState.prepareInstallation).toHaveBeenCalledWith("npm"); + expect(support.testState.bootstrapWorker).toHaveBeenCalledWith({ + operationId: result.provisionOperationId, + sshEndpoint: support.SSH_ENDPOINT, + installation: support.NPM_ARTIFACT, + resolveIdentity: expect.any(Function), + signal: expect.any(AbortSignal), + }); + }); + + it("orphans unknown active leases and adopts unknown expected teardown", async () => { + support.seedReady("worker-unknown"); + support.seedReady("worker-transient"); + support.seedReady("worker-destroyed-unknown"); + support.testState.store.requestDestroy({ + environmentId: "worker-destroyed-unknown", + state: "ready", + }); + support.testState.store.transition({ + environmentId: "worker-destroyed-unknown", + from: "ready", + to: "draining", + }); + support.testState.store.transition({ + environmentId: "worker-destroyed-unknown", + from: "draining", + to: "destroying", + }); + const provider = support.createProvider({ + inspect: async ({ leaseId }) => { + if (leaseId !== "lease:worker-transient") { + return { status: "unknown" }; + } + throw new Error("provider temporarily unavailable"); + }, + }); + const failedTunnelStops = new Set(); + const tunnelManager = { + start: vi.fn(), + stop: vi.fn(async (environmentId: string) => { + if (!failedTunnelStops.has(environmentId)) { + failedTunnelStops.add(environmentId); + throw new Error("tunnel stop interrupted"); + } + }), + stopAll: vi.fn(async () => {}), + status: () => "connected" as const, + } as unknown as WorkerTunnelManager; + const workerService = support.createService(provider, { tunnelManager }); + const admitted = await workerService.admitWorker(support.admissionFor("worker-unknown")); + if (!admitted.ok) { + throw new Error("fixture worker admission failed"); + } + + await workerService.reconcileOnce(); + + expect(support.testState.store.get("worker-unknown")?.state).toBe("draining"); + expect(support.testState.store.get("worker-destroyed-unknown")?.state).toBe("destroying"); + expect(workerService.validateWorkerConnection(admitted.identity)).toBe("credential-replaced"); + expect(support.testState.store.get("worker-transient")).toMatchObject({ + state: "ready", + lastError: "provider temporarily unavailable", + }); + await workerService.reconcileOnce(); + expect(tunnelManager.stop).toHaveBeenCalledTimes(4); + expect(support.testState.store.get("worker-unknown")?.state).toBe("orphaned"); + expect(support.testState.store.get("worker-destroyed-unknown")).toMatchObject({ + state: "destroyed", + }); + }); + + it.each([ + null, + { status: "future" }, + { status: "active", sharedHost: "yes" }, + { status: "unknown", sharedHost: true }, + ])("retains retryable state for malformed inspection result %#", async (inspection) => { + support.seedReady("worker-malformed"); + const provider = support.createProvider({ inspect: async () => inspection as never }); + + await support.createService(provider).reconcileOnce(); + + expect(support.testState.store.get("worker-malformed")).toMatchObject({ + state: "ready", + lastError: expect.stringContaining("invalid inspection"), + }); + }); + + it("records provider-proven teardown without local intent as a failure", async () => { + support.seedReady("worker-destroyed-ready"); + support.seedReady("worker-destroyed-attached"); + support.testState.store.transition({ + environmentId: "worker-destroyed-attached", + from: "ready", + to: "attached", + patch: support.attachedPatch("worker-destroyed-attached", "session-1"), + }); + support.seedReady("worker-destroyed-draining"); + support.testState.store.transition({ + environmentId: "worker-destroyed-draining", + from: "ready", + to: "draining", + }); + const provider = support.createProvider({ + inspect: async () => ({ status: "destroyed" }), + destroy: async () => { + throw new Error("destroy must not run for provider-proven teardown"); + }, + }); + + const workerService = support.createService(provider); + await workerService.reconcileOnce(); + + for (const environmentId of [ + "worker-destroyed-ready", + "worker-destroyed-attached", + "worker-destroyed-draining", + ]) { + expect(support.testState.store.get(environmentId)).toMatchObject({ + state: "failed", + attachedSessionIds: [], + lastError: "Worker environment disappeared before teardown was requested", + }); + } + }); + + it("adopts a provider-proven bootstrap teardown as failed after restart", async () => { + const bootstrapping = support.seedBootstrapping("worker-bootstrap-teardown-crash"); + const requested = support.testState.store.requestDestroy({ + environmentId: bootstrapping.environmentId, + state: bootstrapping.state, + terminalState: "failed", + lastError: "remote bootstrap failed", + }); + const draining = support.testState.store.transition({ + environmentId: requested.environmentId, + from: requested.state, + to: "draining", + patch: { lastError: requested.lastError }, + }); + support.testState.store.transition({ + environmentId: draining.environmentId, + from: draining.state, + to: "destroying", + patch: { lastError: draining.lastError }, + }); + const destroy = vi.fn(async () => {}); + const provider = support.createProvider({ + inspect: async () => ({ status: "destroyed" }), + destroy, + }); + support.testState.providersEnabled = false; + const workerService = support.createService(provider); + + await workerService.reconcileOnce(); + expect(support.testState.store.get(bootstrapping.environmentId)).toMatchObject({ + state: "destroying", + lastError: "remote bootstrap failed", + }); + + support.testState.providersEnabled = true; + await workerService.reconcileOnce(); + + expect(destroy).not.toHaveBeenCalled(); + expect(support.testState.store.get(bootstrapping.environmentId)).toMatchObject({ + state: "failed", + leaseId: null, + sshEndpoint: null, + teardownTerminalState: "failed", + lastError: "remote bootstrap failed", + }); + }); + + it("keeps a failed destroy retryable and makes completed destroy idempotent", async () => { + support.seedReady("worker-destroy"); + support.testState.config.cloudWorkers!.profiles = {}; + let fail = true; + const destroyed: WorkerLifecycleLease[] = []; + const provider = support.createProvider({ + destroy: async (lease) => { + destroyed.push(lease); + if (fail) { + throw new Error("destroy timeout"); + } + }, + }); + const workerService = support.createService(provider); + + await expect(workerService.destroy("worker-destroy")).rejects.toMatchObject({ + code: "provider_failure", + } satisfies Partial); + expect(support.testState.store.get("worker-destroy")).toMatchObject({ + state: "destroying", + lastError: "destroy timeout", + }); + + fail = false; + await workerService.reconcileOnce(); + expect(support.testState.store.get("worker-destroy")).toMatchObject({ state: "destroyed" }); + await workerService.destroy("worker-destroy"); + expect(destroyed).toEqual([ + { leaseId: "lease:worker-destroy", profile: { region: "test" } }, + { leaseId: "lease:worker-destroy", profile: { region: "test" } }, + ]); + }); + + it("adopts an unpersisted provision result before destroying", async () => { + const intent = support.testState.store.createIntent({ + environmentId: "worker-pending-destroy", + providerId: "fake", + profileId: "development", + profileSnapshot: { settings: { region: "test" } }, + provisionOperationId: "provision:pending-destroy", + }); + support.testState.store.transition({ + environmentId: intent.environmentId, + from: "requested", + to: "provisioning", + }); + const destroyed: WorkerLifecycleLease[] = []; + const provider = support.createProvider({ + provision: async () => { + expect( + support.testState.store.get(intent.environmentId)?.destroyRequestedAtMs, + ).not.toBeNull(); + return { leaseId: "lease-1", ssh: support.SSH_ENDPOINT }; + }, + destroy: async (lease) => void destroyed.push(lease), + }); + + const result = await support.createService(provider).destroy(intent.environmentId); + + expect(result.state).toBe("destroyed"); + expect(destroyed).toEqual([{ leaseId: "lease-1", profile: { region: "test" } }]); + }); + + it("retains teardown intent across an indeterminate provision failure", async () => { + support.testState.prepareInstallation = vi.fn(async () => { + throw new Error("bundle preparation must not block teardown adoption"); + }); + const intent = support.testState.store.createIntent({ + environmentId: "worker-pending-destroy-retry", + providerId: "fake", + profileId: "development", + profileSnapshot: { settings: { region: "test" } }, + provisionOperationId: "provision:pending-destroy-retry", + }); + support.testState.store.transition({ + environmentId: intent.environmentId, + from: "requested", + to: "provisioning", + }); + let provisionFails = true; + const destroyed: WorkerLifecycleLease[] = []; + const provider = support.createProvider({ + provision: async () => { + if (provisionFails) { + throw new Error("provision outcome unknown"); + } + return { leaseId: "lease-retried", ssh: support.SSH_ENDPOINT }; + }, + destroy: async (lease) => void destroyed.push(lease), + }); + const workerService = support.createService(provider); + + support.testState.providersEnabled = false; + await expect(workerService.destroy(intent.environmentId)).rejects.toMatchObject({ + code: "provider_not_found", + } satisfies Partial); + expect(support.testState.store.get(intent.environmentId)?.destroyRequestedAtMs).not.toBeNull(); + + support.testState.providersEnabled = true; + await expect(workerService.destroy(intent.environmentId)).rejects.toMatchObject({ + code: "provider_failure", + } satisfies Partial); + expect(support.testState.store.get(intent.environmentId)).toMatchObject({ + state: "provisioning", + destroyRequestedAtMs: expect.any(Number), + }); + + provisionFails = false; + await workerService.reconcileOnce(); + expect(support.testState.store.get(intent.environmentId)?.state).toBe("destroyed"); + expect(destroyed).toEqual([{ leaseId: "lease-retried", profile: { region: "test" } }]); + expect(support.testState.prepareInstallation).not.toHaveBeenCalled(); + }); +}); diff --git a/src/gateway/worker-environments/service-contract.ts b/src/gateway/worker-environments/service-contract.ts index 39f78e67d153..fedfff02d9e1 100644 --- a/src/gateway/worker-environments/service-contract.ts +++ b/src/gateway/worker-environments/service-contract.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import type { WorkerDesktopApp, WorkerProfile } from "../../plugins/capability-provider.types.js"; import type { WorkerSessionPlacementRecord } from "./placement-record.js"; import type { WorkerEnvironmentState } from "./state.js"; @@ -7,6 +8,17 @@ import type { WorkerTunnelStatus, } from "./tunnel-contract.js"; +export function deriveEnvironmentIntent(idempotencyKey: string): { + environmentId: string; + provisionOperationId: string; +} { + const digest = createHash("sha256").update(idempotencyKey).digest("hex"); + return { + environmentId: `worker:${digest.slice(0, 32)}`, + provisionOperationId: `provision:v2:${digest}`, + }; +} + /** Non-secret worker projection available to Gateway request handlers. */ export type WorkerEnvironmentServiceRecord = { environmentId: string; diff --git a/src/gateway/worker-environments/service-lifetime.test.ts b/src/gateway/worker-environments/service-lifetime.test.ts new file mode 100644 index 000000000000..4e23b7ac463d --- /dev/null +++ b/src/gateway/worker-environments/service-lifetime.test.ts @@ -0,0 +1,226 @@ +import { describe, expect, it, vi } from "vitest"; +import * as support from "./service.test-support.js"; +import type { WorkerTunnelManager } from "./tunnel.js"; + +type WorkerEnvironmentServiceError = support.WorkerEnvironmentServiceError; +type WorkerLifecycleLease = support.WorkerLifecycleLease; + +describe("worker environment service", () => { + support.setupWorkerEnvironmentServiceSuite(); + + it("reconciles unrelated leases concurrently", async () => { + support.seedReady("worker-concurrent-a"); + support.seedReady("worker-concurrent-b"); + let release: (() => void) | undefined; + const blocked = new Promise((resolve) => { + release = resolve; + }); + const inspected: WorkerLifecycleLease[] = []; + const provider = support.createProvider({ + inspect: async (lease) => { + inspected.push(lease); + await blocked; + return { status: "active" }; + }, + }); + + const reconciliation = support.createService(provider).reconcileOnce(); + try { + await support.waitForFast(() => expect(inspected).toHaveLength(2)); + } finally { + release?.(); + } + await reconciliation; + + expect(new Set(inspected.map(({ leaseId }) => leaseId))).toEqual( + new Set(["lease:worker-concurrent-a", "lease:worker-concurrent-b"]), + ); + }); + + it("waits for timed-out provider work during shutdown", async () => { + let finishProvision: (() => void) | undefined; + const provisionPending = new Promise((resolve) => { + finishProvision = resolve; + }); + const provision = vi.fn(async () => { + await provisionPending; + return { leaseId: "lease-stop-timeout", ssh: support.SSH_ENDPOINT }; + }); + const stopAll = vi.fn(async () => {}); + const tunnelManager = { + stopAll, + } as unknown as WorkerTunnelManager; + const workerService = support.createService(support.createProvider({ provision }), { + providerCallTimeoutMs: 5, + tunnelManager, + }); + const creation = workerService.create("development", "request-stop-provider-timeout"); + const creationResult = expect(creation).rejects.toMatchObject({ + code: "provider_failure", + } satisfies Partial); + let stopped = false; + let stopping: Promise | undefined; + + try { + await support.waitForFast(() => expect(provision).toHaveBeenCalledOnce()); + await creationResult; + stopping = workerService.stop().then(() => { + stopped = true; + }); + await support.waitForFast(() => expect(stopAll).toHaveBeenCalledOnce()); + await Promise.resolve(); + expect(stopped).toBe(false); + } finally { + finishProvision?.(); + } + + await stopping; + expect(stopped).toBe(true); + }); + + it("owns and clears one periodic reconciliation timer", async () => { + vi.useFakeTimers(); + const liveEvents = support.createLiveEvents(); + const workerService = support.createService(support.createProvider(), { liveEvents }); + + workerService.start(); + workerService.start(); + expect(liveEvents.start).toHaveBeenCalledOnce(); + expect(vi.getTimerCount()).toBe(1); + await workerService.stop(); + + expect(liveEvents.clear).toHaveBeenCalledTimes(2); + expect(vi.getTimerCount()).toBe(0); + }); + + it("rejects a create queued before service shutdown once its lock is acquired", async () => { + let finishBootstrap: (() => void) | undefined; + const bootstrapPending = new Promise((resolve) => { + finishBootstrap = resolve; + }); + support.testState.bootstrapWorker = vi.fn(async () => { + await bootstrapPending; + return support.BOOTSTRAP_RECEIPT; + }); + const provision = vi.fn(support.createProvider().provision); + const workerService = support.createService(support.createProvider({ provision })); + const first = workerService.create("development", "request-queued-before-stop"); + await support.waitForFast(() => + expect(support.testState.bootstrapWorker).toHaveBeenCalledTimes(1), + ); + const queued = workerService.create("development", "request-queued-before-stop"); + const queuedResult = expect(queued).rejects.toMatchObject({ + code: "invalid_state", + } satisfies Partial); + + const stopping = workerService.stop(); + finishBootstrap?.(); + + await expect(first).resolves.toMatchObject({ state: "ready" }); + await queuedResult; + await stopping; + expect(provision).toHaveBeenCalledTimes(1); + }); + + it("drains a destroy accepted before service shutdown while it waits for the lock", async () => { + let finishBootstrap: (() => void) | undefined; + const bootstrapPending = new Promise((resolve) => { + finishBootstrap = resolve; + }); + support.testState.bootstrapWorker = vi.fn(async () => { + await bootstrapPending; + return support.BOOTSTRAP_RECEIPT; + }); + const destroy = vi.fn(async () => {}); + const workerService = support.createService(support.createProvider({ destroy })); + const creation = workerService.create("development", "request-destroy-before-stop"); + await support.waitForFast(() => + expect(support.testState.bootstrapWorker).toHaveBeenCalledTimes(1), + ); + const environmentId = support.testState.store.list()[0]?.environmentId; + expect(environmentId).toBeTruthy(); + const teardown = workerService.destroy(environmentId!); + const teardownResult = expect(teardown).resolves.toMatchObject({ state: "destroyed" }); + + const stopping = workerService.stop(); + finishBootstrap?.(); + + await expect(creation).resolves.toMatchObject({ state: "ready" }); + await teardownResult; + await stopping; + expect(destroy).toHaveBeenCalledTimes(1); + }); + + it("drains accepted operations after reconciliation rejects during shutdown", async () => { + const durableStore = support.testState.store; + support.testState.store = { + ...support.testState.store, + listForReconcile() { + throw new Error("reconcile database read failed"); + }, + }; + let finishBootstrap: (() => void) | undefined; + const bootstrapPending = new Promise((resolve) => { + finishBootstrap = resolve; + }); + support.testState.bootstrapWorker = vi.fn(async () => { + await bootstrapPending; + return support.BOOTSTRAP_RECEIPT; + }); + const workerService = support.createService(support.createProvider()); + const creation = workerService.create("development", "request-stop-after-reconcile-failure"); + await support.waitForFast(() => + expect(support.testState.bootstrapWorker).toHaveBeenCalledTimes(1), + ); + const reconciliation = workerService.reconcileOnce(); + const reconciliationResult = expect(reconciliation).rejects.toThrow( + "reconcile database read failed", + ); + let stopped = false; + const stopping = workerService.stop().then(() => { + stopped = true; + }); + + await reconciliationResult; + await Promise.resolve(); + expect(stopped).toBe(false); + finishBootstrap?.(); + + await expect(creation).resolves.toMatchObject({ state: "ready" }); + await stopping; + expect(stopped).toBe(true); + expect(durableStore.list()).toHaveLength(1); + }); + + it("starts without blocking gateway startup and drains reconciliation on stop", async () => { + support.seedReady("worker-slow-inspection"); + let finishInspection: (() => void) | undefined; + const inspectionPending = new Promise((resolve) => { + finishInspection = resolve; + }); + const inspect = vi.fn(async () => { + await inspectionPending; + return { status: "active" as const }; + }); + const workerService = support.createService(support.createProvider({ inspect })); + + workerService.start(); + await support.waitForFast(() => expect(inspect).toHaveBeenCalledTimes(1)); + let stopped = false; + const stopping = workerService.stop().then(() => { + stopped = true; + }); + await Promise.resolve(); + expect(stopped).toBe(false); + + finishInspection?.(); + await stopping; + expect(stopped).toBe(true); + await expect(workerService.create("development", "request-after-stop")).rejects.toMatchObject({ + code: "invalid_state", + } satisfies Partial); + await expect(workerService.destroy("worker-slow-inspection")).rejects.toMatchObject({ + code: "invalid_state", + } satisfies Partial); + }); +}); diff --git a/src/gateway/worker-environments/service.test-support.ts b/src/gateway/worker-environments/service.test-support.ts new file mode 100644 index 000000000000..91689ec630ef --- /dev/null +++ b/src/gateway/worker-environments/service.test-support.ts @@ -0,0 +1,458 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { expectDefined } from "@openclaw/normalization-core"; +import { afterEach, beforeEach, vi } from "vitest"; +import type { OpenClawConfig } from "../../config/types.js"; +import type { + WorkerDesktopEndpoint, + WorkerProvider, + WorkerSshEndpoint, +} from "../../plugins/types.js"; +import { + closeOpenClawStateDatabaseForTest, + openOpenClawStateDatabase, + type OpenClawStateDatabase, +} from "../../state/openclaw-state-db.js"; +import type { WorkerInstallationArtifact } from "./bundle.js"; +import type { WorkerConnectionIdentity } from "./connection-identity.js"; +import { hashWorkerCredential } from "./credential.js"; +import { createWorkerInferenceStore } from "./inference-store.js"; +import { createWorkerEnvironmentService, type WorkerEnvironmentService } from "./service.js"; +import { createWorkerEnvironmentStore, type WorkerEnvironmentStore } from "./store.js"; + +export function waitForFast( + callback: () => T | Promise, + options: { timeout?: number; interval?: number } = {}, +) { + return vi.waitFor(callback, { interval: 1, ...options }); +} + +const HOST_KEY = [["ssh", "ed25519"].join("-"), "AAAA"].join(" "); +export type WorkerEnvironmentServiceOptions = Parameters[0]; +export type WorkerEnvironmentServiceError = Error & { code: string }; +export const SSH_ENDPOINT: WorkerSshEndpoint = { + host: "worker.example.test", + port: 22, + user: "openclaw", + hostKey: HOST_KEY, + keyRef: { source: "file", provider: "worker-keys", id: "/development-key" }, +}; +export const DESKTOP: WorkerDesktopEndpoint = { + protocol: "rfb", + port: 5900, + passwordFilePath: "/var/lib/crabbox/vnc.password", + apps: [ + { + id: "browser", + executablePath: "/usr/local/bin/openclaw-worker-browser", + cdpPort: 9222, + }, + { id: "terminal", executablePath: "/usr/local/bin/openclaw-worker-terminal" }, + ], +}; +export const BUNDLE_HASH = "a".repeat(64); +export const BUNDLE_ARTIFACT: WorkerInstallationArtifact = { + install: "bundle", + bundleHash: BUNDLE_HASH, + openclawVersion: "2026.7.2", + protocolFeatures: [], + tarballSha256: "b".repeat(64), + tarballPath: "/gateway/cache/worker-bundle.tgz", +}; +export const NPM_ARTIFACT: WorkerInstallationArtifact = { + install: "npm", + bundleHash: BUNDLE_HASH, + openclawVersion: "2026.7.2", + packageIntegrity: `sha512-${Buffer.alloc(64).toString("base64")}`, + protocolFeatures: [], + packageSpec: "openclaw@2026.7.2", +}; +export const BOOTSTRAP_RECEIPT = { + bundleHash: BUNDLE_HASH, + openclawVersion: "2026.7.2", + protocolFeatures: [], +}; +export const CREDENTIAL = ["worker", "credential", "fixture"].join("-"); +const LIVE_EVENT_ACK = { ok: true as const, result: { ackedSeq: 1 } }; +export const LIVE_EVENT = { + runEpoch: 1, + lastAckedSeq: 0, + seq: 1, + runId: "run-1", + event: { kind: "assistant" as const, payload: { text: "hi", delta: "hi" } }, +}; + +export type WorkerLifecycleLease = Parameters[0]; +type TranscriptRequest = Parameters[1]; +type TranscriptOverrides = Partial>; +type LiveEventRequest = Parameters[1]; +type LiveOpts = Partial>; + +export const testState = {} as { + root: string; + stateDb: OpenClawStateDatabase; + store: WorkerEnvironmentStore; + service: WorkerEnvironmentService | undefined; + config: OpenClawConfig; + nowMs: number; + providersEnabled: boolean; + prepareInstallation: WorkerEnvironmentServiceOptions["prepareInstallation"]; + bootstrapWorker: WorkerEnvironmentServiceOptions["bootstrapWorker"]; +}; + +export function setupWorkerEnvironmentServiceSuite() { + beforeEach(async () => { + testState.root = await fs.mkdtemp( + path.join(await fs.realpath(os.tmpdir()), "openclaw-worker-service-"), + ); + testState.stateDb = openOpenClawStateDatabase({ + env: { OPENCLAW_STATE_DIR: testState.root }, + }); + testState.nowMs = 1_000; + testState.providersEnabled = true; + testState.store = createWorkerEnvironmentStore({ + database: testState.stateDb, + now: () => testState.nowMs, + }); + testState.config = { + cloudWorkers: { + desktop: true, + profiles: { + development: { + provider: "fake", + settings: { region: "test" }, + }, + }, + }, + }; + testState.prepareInstallation = vi.fn(async (install) => + install === "bundle" ? BUNDLE_ARTIFACT : NPM_ARTIFACT, + ); + testState.bootstrapWorker = vi.fn(async ({ installation }) => ({ + bundleHash: installation.bundleHash, + openclawVersion: installation.openclawVersion, + protocolFeatures: [...installation.protocolFeatures], + })); + }); + + afterEach(async () => { + await testState.service?.stop(); + vi.useRealTimers(); + closeOpenClawStateDatabaseForTest(); + await fs.rm(testState.root, { recursive: true, force: true }); + }); +} + +export function getDevelopmentProfile() { + return expectDefined( + testState.config.cloudWorkers?.profiles?.development, + "development worker profile", + ); +} + +export function createService( + provider: WorkerProvider, + serviceOptions: Partial< + Pick< + WorkerEnvironmentServiceOptions, + | "applyTranscriptCommit" + | "bootstrapCallTimeoutMs" + | "executeInference" + | "providerCallTimeoutMs" + | "resolveSshIdentity" + | "resolveWorkerGateway" + | "tunnelManager" + | "generateWorkerCredential" + | "liveEvents" + | "placementStore" + | "workerCredentialTtlMs" + > + > = {}, +) { + testState.service = createWorkerEnvironmentService({ + store: testState.store, + getConfig: () => testState.config, + resolveProvider: (providerId) => + testState.providersEnabled && providerId === "fake" ? provider : undefined, + prepareInstallation: testState.prepareInstallation, + bootstrapWorker: testState.bootstrapWorker, + resolveSshIdentity: async () => ({ kind: "path", path: "/keys/worker" }), + resolveWorkerGateway: () => ({ host: "127.0.0.1", port: 18_789 }), + generateWorkerCredential: () => CREDENTIAL, + executeInference: async () => ({ + type: "error", + reason: "cancelled", + message: "Inference cancelled", + }), + inferenceStore: createWorkerInferenceStore({ + database: testState.stateDb, + now: () => testState.nowMs, + }), + now: () => testState.nowMs, + reconcileIntervalMs: 25, + ...serviceOptions, + }); + return testState.service; +} + +export function createProvider(overrides: Partial = {}): WorkerProvider { + return { + id: "fake", + provision: async () => ({ leaseId: "lease-1", ssh: SSH_ENDPOINT }), + inspect: async () => ({ status: "active" }), + destroy: async () => {}, + ...overrides, + }; +} + +export function createLiveEvents(overrides: Record = {}) { + return { + apply: vi.fn(() => LIVE_EVENT_ACK), + bindSession: vi.fn(() => true), + clear: vi.fn(), + clearEnvironment: vi.fn(), + rotateCredential: vi.fn(() => true), + start: vi.fn(), + ...overrides, + }; +} + +export function seedBootstrapping( + environmentId: string, + install?: WorkerInstallationArtifact["install"], + sharedHost = false, +) { + const intent = testState.store.createIntent({ + environmentId, + providerId: "fake", + profileId: "development", + profileSnapshot: { ...(install ? { install } : {}), settings: { region: "test" } }, + provisionOperationId: `provision:${environmentId}`, + }); + const provisioning = testState.store.transition({ + environmentId, + from: intent.state, + to: "provisioning", + }); + return testState.store.transition({ + environmentId, + from: provisioning.state, + to: "bootstrapping", + patch: { leaseId: `lease:${environmentId}`, sshEndpoint: SSH_ENDPOINT, sharedHost }, + }); +} + +export function seedReady( + environmentId: string, + install?: WorkerInstallationArtifact["install"], + sharedHost = false, +) { + const bootstrapping = seedBootstrapping(environmentId, install, sharedHost); + return testState.store.transition({ + environmentId, + from: bootstrapping.state, + to: "ready", + patch: readyPatch(environmentId), + }); +} + +export function seedReadyDesktop(environmentId: string, desktop: WorkerDesktopEndpoint = DESKTOP) { + const intent = testState.store.createIntent({ + environmentId, + providerId: "fake", + profileId: "development", + profileSnapshot: { settings: { region: "test", desktop: true } }, + provisionOperationId: `provision:${environmentId}`, + }); + const provisioning = testState.store.transition({ + environmentId, + from: intent.state, + to: "provisioning", + }); + const bootstrapping = testState.store.transition({ + environmentId, + from: provisioning.state, + to: "bootstrapping", + patch: { + leaseId: `lease:${environmentId}`, + sshEndpoint: SSH_ENDPOINT, + desktop, + }, + }); + return testState.store.transition({ + environmentId, + from: bootstrapping.state, + to: "ready", + patch: readyPatch(environmentId), + }); +} + +export function readyPatch(environmentId: string, receipt = BOOTSTRAP_RECEIPT) { + return { + bootstrapReceipt: receipt, + credential: { + credentialHash: hashWorkerCredential([CREDENTIAL, environmentId].join("-")), + sessionId: null, + rpcSetVersion: 1, + expiresAtMs: testState.nowMs + 10_000, + }, + }; +} + +export function attachedPatch(environmentId: string, sessionId: string) { + return { + attachedSessionIds: [sessionId], + credential: { + credentialHash: hashWorkerCredential([CREDENTIAL, environmentId, sessionId].join("-")), + sessionId, + rpcSetVersion: 1, + expiresAtMs: testState.nowMs + 10_000, + }, + }; +} + +export function admissionFor(environmentId: string) { + return { + environmentId, + credential: [CREDENTIAL, environmentId].join("-"), + sessionId: null, + runId: null, + ownerEpoch: 1, + rpcSetVersion: 1, + handshake: BOOTSTRAP_RECEIPT, + }; +} + +export function seedAttachedIdentity( + environmentId: string, + sessionId: string, +): WorkerConnectionIdentity { + const ready = seedReady(environmentId); + const attached = testState.store.transition({ + environmentId, + from: ready.state, + to: "attached", + patch: attachedPatch(environmentId, sessionId), + }); + const credential = testState.store.getCredential(environmentId); + if (!credential || !attached.bootstrapReceipt) { + throw new Error("attached worker fixture is incomplete"); + } + return { + environmentId, + credentialHash: credential.credentialHash, + bundleHash: credential.bundleHash, + sessionId, + runId: "run-1", + ownerEpoch: attached.ownerEpoch, + rpcSetVersion: credential.rpcSetVersion, + protocolFeatures: [...attached.bootstrapReceipt.protocolFeatures], + credentialExpiresAtMs: credential.expiresAtMs, + }; +} + +export function inferenceRequest( + identity: WorkerConnectionIdentity, +): Parameters[1] { + return { + runEpoch: identity.ownerEpoch, + sessionId: identity.sessionId ?? "session-missing", + runId: identity.runId ?? "run-missing", + turnId: "turn-inference", + modelRef: { provider: "fake", model: "model-test" }, + context: { messages: [] }, + options: {}, + }; +} + +export function transcriptRequest( + identity: WorkerConnectionIdentity, + text: string, + overrides: TranscriptOverrides = {}, +): TranscriptRequest { + return { + runEpoch: identity.ownerEpoch, + seq: 1, + baseLeafId: null, + messages: [ + { + role: "user", + content: [{ type: "text", text }], + timestamp: 1, + }, + ], + ...overrides, + }; +} + +function liveEventRequest( + identity: WorkerConnectionIdentity, + event: LiveEventRequest["event"], + overrides: LiveOpts = {}, +): LiveEventRequest { + return { + runEpoch: identity.ownerEpoch, + lastAckedSeq: 0, + seq: 1, + runId: identity.runId ?? "run-missing", + event, + ...overrides, + }; +} + +export function assistantEvent( + identity: WorkerConnectionIdentity, + text: string, + extra: LiveOpts = {}, +) { + return liveEventRequest(identity, { kind: "assistant", payload: { text, delta: text } }, extra); +} + +export function terminalEvent(identity: WorkerConnectionIdentity, overrides: LiveOpts = {}) { + return liveEventRequest( + identity, + { kind: "lifecycle", payload: { phase: "end", endedAt: 2 } }, + overrides, + ); +} + +export function successfulTranscriptCommit(entryId: string, beforeCommit?: () => Promise) { + return vi.fn(async () => { + await beforeCommit?.(); + return { ok: true as const, result: { entryIds: [entryId], newLeafId: entryId } }; + }); +} + +export function sequencedLiveEvents(ackedSeq = (seq: number) => seq) { + const apply = vi.fn(({ request }: { request: LiveEventRequest }) => ({ + ok: true as const, + result: { ackedSeq: ackedSeq(request.seq) }, + })); + return { apply, liveEvents: createLiveEvents({ apply }) }; +} + +export function placementBinding(identity: WorkerConnectionIdentity) { + return { + sessionId: identity.sessionId ?? "session-missing", + environmentId: identity.environmentId, + ownerEpoch: identity.ownerEpoch, + runId: identity.runId ?? "run-missing", + }; +} + +export function placementHarness( + environmentId: string, + sessionId: string, + serviceOptions: Parameters[1] = {}, +) { + const identity = seedAttachedIdentity(environmentId, sessionId); + const placementStore = { + hasWorkerTurn: vi.fn(() => true), + validateWorkerTurn: vi.fn(() => true), + isWorkerTurnToolAuthorized: vi.fn(() => true), + updateAckCursors: vi.fn(), + }; + const workerService = createService(createProvider(), { ...serviceOptions, placementStore }); + return { identity, placementStore, workerService }; +} diff --git a/src/gateway/worker-environments/service.test.ts b/src/gateway/worker-environments/service.test.ts deleted file mode 100644 index 78ae05ba4f55..000000000000 --- a/src/gateway/worker-environments/service.test.ts +++ /dev/null @@ -1,3165 +0,0 @@ -import fs from "node:fs/promises"; -import os from "node:os"; -import path from "node:path"; -import { expectDefined } from "@openclaw/normalization-core"; -import { MAX_TIMER_TIMEOUT_MS } from "@openclaw/normalization-core/number-coercion"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import type { OpenClawConfig } from "../../config/types.js"; -import { - WorkerProviderError, - type WorkerDesktopEndpoint, - type WorkerProfile, - type WorkerProvider, - type WorkerSshEndpoint, -} from "../../plugins/types.js"; -import { - closeOpenClawStateDatabaseForTest, - openOpenClawStateDatabase, - type OpenClawStateDatabase, -} from "../../state/openclaw-state-db.js"; -import type { GatewaySessionRow } from "../session-utils.types.js"; -import { writeSessionStore } from "../test-helpers.js"; -import { directSessionReq } from "../test/server-sessions.test-helpers.js"; -import type { WorkerInstallationArtifact } from "./bundle.js"; -import type { WorkerConnectionIdentity } from "./connection-identity.js"; -import { hashWorkerCredential } from "./credential.js"; -import { createWorkerInferenceStore } from "./inference-store.js"; -import { createWorkerPlacementDispatchService } from "./placement-dispatch.js"; -import { createWorkerSessionPlacementStore } from "./placement-store.js"; -import { createWorkerEnvironmentService, type WorkerEnvironmentService } from "./service.js"; -import { createWorkerEnvironmentStore, type WorkerEnvironmentStore } from "./store.js"; -import type { WorkerTunnelManager } from "./tunnel.js"; -import { createWorkerWorkspaceOperationCoordinator } from "./workspace-operation-coordinator.js"; - -function waitForFast( - callback: () => T | Promise, - options: { timeout?: number; interval?: number } = {}, -) { - return vi.waitFor(callback, { interval: 1, ...options }); -} - -const HOST_KEY = [["ssh", "ed25519"].join("-"), "AAAA"].join(" "); -type WorkerEnvironmentServiceOptions = Parameters[0]; -type WorkerEnvironmentServiceError = Error & { code: string }; -const SSH_ENDPOINT: WorkerSshEndpoint = { - host: "worker.example.test", - port: 22, - user: "openclaw", - hostKey: HOST_KEY, - keyRef: { source: "file", provider: "worker-keys", id: "/development-key" }, -}; -const DESKTOP: WorkerDesktopEndpoint = { - protocol: "rfb", - port: 5900, - passwordFilePath: "/var/lib/crabbox/vnc.password", - apps: [ - { - id: "browser", - executablePath: "/usr/local/bin/openclaw-worker-browser", - cdpPort: 9222, - }, - { id: "terminal", executablePath: "/usr/local/bin/openclaw-worker-terminal" }, - ], -}; -const BUNDLE_HASH = "a".repeat(64); -const BUNDLE_ARTIFACT: WorkerInstallationArtifact = { - install: "bundle", - bundleHash: BUNDLE_HASH, - openclawVersion: "2026.7.2", - protocolFeatures: [], - tarballSha256: "b".repeat(64), - tarballPath: "/gateway/cache/worker-bundle.tgz", -}; -const NPM_ARTIFACT: WorkerInstallationArtifact = { - install: "npm", - bundleHash: BUNDLE_HASH, - openclawVersion: "2026.7.2", - packageIntegrity: `sha512-${Buffer.alloc(64).toString("base64")}`, - protocolFeatures: [], - packageSpec: "openclaw@2026.7.2", -}; -const BOOTSTRAP_RECEIPT = { - bundleHash: BUNDLE_HASH, - openclawVersion: "2026.7.2", - protocolFeatures: [], -}; -const CREDENTIAL = ["worker", "credential", "fixture"].join("-"); -const LIVE_EVENT_ACK = { ok: true as const, result: { ackedSeq: 1 } }; -const LIVE_EVENT = { - runEpoch: 1, - lastAckedSeq: 0, - seq: 1, - runId: "run-1", - event: { kind: "assistant" as const, payload: { text: "hi", delta: "hi" } }, -}; - -type WorkerLifecycleLease = Parameters[0]; -type TranscriptRequest = Parameters[1]; -type TranscriptOverrides = Partial>; -type LiveEventRequest = Parameters[1]; -type LiveOpts = Partial>; - -describe("worker environment service", () => { - let root: string; - let database: OpenClawStateDatabase; - let store: WorkerEnvironmentStore; - let service: WorkerEnvironmentService | undefined; - let config: OpenClawConfig; - let nowMs: number; - let providersEnabled: boolean; - let prepareInstallation: WorkerEnvironmentServiceOptions["prepareInstallation"]; - let bootstrapWorker: WorkerEnvironmentServiceOptions["bootstrapWorker"]; - - beforeEach(async () => { - root = await fs.mkdtemp(path.join(await fs.realpath(os.tmpdir()), "openclaw-worker-service-")); - database = openOpenClawStateDatabase({ env: { OPENCLAW_STATE_DIR: root } }); - nowMs = 1_000; - providersEnabled = true; - store = createWorkerEnvironmentStore({ database, now: () => nowMs }); - config = { - cloudWorkers: { - desktop: true, - profiles: { - development: { - provider: "fake", - settings: { region: "test" }, - }, - }, - }, - }; - prepareInstallation = vi.fn(async (install) => - install === "bundle" ? BUNDLE_ARTIFACT : NPM_ARTIFACT, - ); - bootstrapWorker = vi.fn(async ({ installation }) => ({ - bundleHash: installation.bundleHash, - openclawVersion: installation.openclawVersion, - protocolFeatures: [...installation.protocolFeatures], - })); - }); - - afterEach(async () => { - await service?.stop(); - vi.useRealTimers(); - closeOpenClawStateDatabaseForTest(); - await fs.rm(root, { recursive: true, force: true }); - }); - - function getDevelopmentProfile() { - return expectDefined(config.cloudWorkers?.profiles?.development, "development worker profile"); - } - - function createService( - provider: WorkerProvider, - serviceOptions: Partial< - Pick< - WorkerEnvironmentServiceOptions, - | "applyTranscriptCommit" - | "bootstrapCallTimeoutMs" - | "executeInference" - | "providerCallTimeoutMs" - | "resolveSshIdentity" - | "resolveWorkerGateway" - | "tunnelManager" - | "generateWorkerCredential" - | "liveEvents" - | "placementStore" - | "workerCredentialTtlMs" - > - > = {}, - ) { - service = createWorkerEnvironmentService({ - store, - getConfig: () => config, - resolveProvider: (providerId) => - providersEnabled && providerId === "fake" ? provider : undefined, - prepareInstallation, - bootstrapWorker, - resolveSshIdentity: async () => ({ kind: "path", path: "/keys/worker" }), - resolveWorkerGateway: () => ({ host: "127.0.0.1", port: 18_789 }), - generateWorkerCredential: () => CREDENTIAL, - executeInference: async () => ({ - type: "error", - reason: "cancelled", - message: "Inference cancelled", - }), - inferenceStore: createWorkerInferenceStore({ database, now: () => nowMs }), - now: () => nowMs, - reconcileIntervalMs: 25, - ...serviceOptions, - }); - return service; - } - - function createProvider(overrides: Partial = {}): WorkerProvider { - return { - id: "fake", - provision: async () => ({ leaseId: "lease-1", ssh: SSH_ENDPOINT }), - inspect: async () => ({ status: "active" }), - destroy: async () => {}, - ...overrides, - }; - } - - function createLiveEvents(overrides: Record = {}) { - return { - apply: vi.fn(() => LIVE_EVENT_ACK), - bindSession: vi.fn(() => true), - clear: vi.fn(), - clearEnvironment: vi.fn(), - rotateCredential: vi.fn(() => true), - start: vi.fn(), - ...overrides, - }; - } - - function seedBootstrapping( - environmentId: string, - install?: WorkerInstallationArtifact["install"], - sharedHost = false, - ) { - const intent = store.createIntent({ - environmentId, - providerId: "fake", - profileId: "development", - profileSnapshot: { ...(install ? { install } : {}), settings: { region: "test" } }, - provisionOperationId: `provision:${environmentId}`, - }); - const provisioning = store.transition({ - environmentId, - from: intent.state, - to: "provisioning", - }); - return store.transition({ - environmentId, - from: provisioning.state, - to: "bootstrapping", - patch: { leaseId: `lease:${environmentId}`, sshEndpoint: SSH_ENDPOINT, sharedHost }, - }); - } - - function seedReady( - environmentId: string, - install?: WorkerInstallationArtifact["install"], - sharedHost = false, - ) { - const bootstrapping = seedBootstrapping(environmentId, install, sharedHost); - return store.transition({ - environmentId, - from: bootstrapping.state, - to: "ready", - patch: readyPatch(environmentId), - }); - } - - function seedReadyDesktop(environmentId: string, desktop: WorkerDesktopEndpoint = DESKTOP) { - const intent = store.createIntent({ - environmentId, - providerId: "fake", - profileId: "development", - profileSnapshot: { settings: { region: "test", desktop: true } }, - provisionOperationId: `provision:${environmentId}`, - }); - const provisioning = store.transition({ - environmentId, - from: intent.state, - to: "provisioning", - }); - const bootstrapping = store.transition({ - environmentId, - from: provisioning.state, - to: "bootstrapping", - patch: { - leaseId: `lease:${environmentId}`, - sshEndpoint: SSH_ENDPOINT, - desktop, - }, - }); - return store.transition({ - environmentId, - from: bootstrapping.state, - to: "ready", - patch: readyPatch(environmentId), - }); - } - - function readyPatch(environmentId: string, receipt = BOOTSTRAP_RECEIPT) { - return { - bootstrapReceipt: receipt, - credential: { - credentialHash: hashWorkerCredential([CREDENTIAL, environmentId].join("-")), - sessionId: null, - rpcSetVersion: 1, - expiresAtMs: nowMs + 10_000, - }, - }; - } - - function attachedPatch(environmentId: string, sessionId: string) { - return { - attachedSessionIds: [sessionId], - credential: { - credentialHash: hashWorkerCredential([CREDENTIAL, environmentId, sessionId].join("-")), - sessionId, - rpcSetVersion: 1, - expiresAtMs: nowMs + 10_000, - }, - }; - } - - function admissionFor(environmentId: string) { - return { - environmentId, - credential: [CREDENTIAL, environmentId].join("-"), - sessionId: null, - runId: null, - ownerEpoch: 1, - rpcSetVersion: 1, - handshake: BOOTSTRAP_RECEIPT, - }; - } - - function seedAttachedIdentity( - environmentId: string, - sessionId: string, - ): WorkerConnectionIdentity { - const ready = seedReady(environmentId); - const attached = store.transition({ - environmentId, - from: ready.state, - to: "attached", - patch: attachedPatch(environmentId, sessionId), - }); - const credential = store.getCredential(environmentId); - if (!credential || !attached.bootstrapReceipt) { - throw new Error("attached worker fixture is incomplete"); - } - return { - environmentId, - credentialHash: credential.credentialHash, - bundleHash: credential.bundleHash, - sessionId, - runId: "run-1", - ownerEpoch: attached.ownerEpoch, - rpcSetVersion: credential.rpcSetVersion, - protocolFeatures: [...attached.bootstrapReceipt.protocolFeatures], - credentialExpiresAtMs: credential.expiresAtMs, - }; - } - - function inferenceRequest( - identity: WorkerConnectionIdentity, - ): Parameters[1] { - return { - runEpoch: identity.ownerEpoch, - sessionId: identity.sessionId ?? "session-missing", - runId: identity.runId ?? "run-missing", - turnId: "turn-inference", - modelRef: { provider: "fake", model: "model-test" }, - context: { messages: [] }, - options: {}, - }; - } - - function transcriptRequest( - identity: WorkerConnectionIdentity, - text: string, - overrides: TranscriptOverrides = {}, - ): TranscriptRequest { - return { - runEpoch: identity.ownerEpoch, - seq: 1, - baseLeafId: null, - messages: [ - { - role: "user", - content: [{ type: "text", text }], - timestamp: 1, - }, - ], - ...overrides, - }; - } - - function liveEventRequest( - identity: WorkerConnectionIdentity, - event: LiveEventRequest["event"], - overrides: LiveOpts = {}, - ): LiveEventRequest { - return { - runEpoch: identity.ownerEpoch, - lastAckedSeq: 0, - seq: 1, - runId: identity.runId ?? "run-missing", - event, - ...overrides, - }; - } - - function assistantEvent(identity: WorkerConnectionIdentity, text: string, extra: LiveOpts = {}) { - return liveEventRequest(identity, { kind: "assistant", payload: { text, delta: text } }, extra); - } - - function terminalEvent(identity: WorkerConnectionIdentity, overrides: LiveOpts = {}) { - return liveEventRequest( - identity, - { kind: "lifecycle", payload: { phase: "end", endedAt: 2 } }, - overrides, - ); - } - - function successfulTranscriptCommit(entryId: string, beforeCommit?: () => Promise) { - return vi.fn(async () => { - await beforeCommit?.(); - return { ok: true as const, result: { entryIds: [entryId], newLeafId: entryId } }; - }); - } - - function sequencedLiveEvents(ackedSeq = (seq: number) => seq) { - const apply = vi.fn(({ request }: { request: LiveEventRequest }) => ({ - ok: true as const, - result: { ackedSeq: ackedSeq(request.seq) }, - })); - return { apply, liveEvents: createLiveEvents({ apply }) }; - } - - function placementBinding(identity: WorkerConnectionIdentity) { - return { - sessionId: identity.sessionId ?? "session-missing", - environmentId: identity.environmentId, - ownerEpoch: identity.ownerEpoch, - runId: identity.runId ?? "run-missing", - }; - } - - function placementHarness( - environmentId: string, - sessionId: string, - serviceOptions: Parameters[1] = {}, - ) { - const identity = seedAttachedIdentity(environmentId, sessionId); - const placementStore = { - hasWorkerTurn: vi.fn(() => true), - validateWorkerTurn: vi.fn(() => true), - isWorkerTurnToolAuthorized: vi.fn(() => true), - updateAckCursors: vi.fn(), - }; - const workerService = createService(createProvider(), { ...serviceOptions, placementStore }); - return { identity, placementStore, workerService }; - } - - it("persists intent and an immutable profile snapshot before provisioning", async () => { - const operationIds: string[] = []; - const provider = createProvider({ - provision: async (profile, operationId) => { - operationIds.push(operationId); - expect(store.list()[0]).toMatchObject({ - state: "provisioning", - provisionOperationId: operationId, - profileSnapshot: { - install: "bundle", - settings: { region: "test" }, - }, - }); - getDevelopmentProfile().settings = { region: "mutated" }; - expect(profile).toEqual({ region: "test" }); - return { leaseId: "lease-1", ssh: SSH_ENDPOINT }; - }, - }); - - const workerService = createService(provider); - const result = await workerService.create("development", "request-1"); - const repeated = await workerService.create("development", "request-1"); - - expect(result).toMatchObject({ state: "ready", leaseId: "lease-1", ownerEpoch: 1 }); - expect(repeated.environmentId).toBe(result.environmentId); - expect(operationIds).toHaveLength(1); - expect(operationIds[0]).toMatch(/^provision:v2:[a-f0-9]{64}$/u); - expect(result.profileSnapshot).toMatchObject({ settings: { region: "test" } }); - expect(store.getCredential(result.environmentId)).toMatchObject({ - credentialHash: hashWorkerCredential(CREDENTIAL), - ownerEpoch: 1, - sessionId: null, - }); - const persistedCredential = database.db - .prepare("SELECT * FROM worker_environment_credentials WHERE environment_id = ?") - .get(result.environmentId); - expect(persistedCredential).toMatchObject({ - credential_hash: hashWorkerCredential(CREDENTIAL), - }); - expect(JSON.stringify(persistedCredential)).not.toContain(CREDENTIAL); - const binding = { environmentId: result.environmentId, ownerEpoch: 1, sessionId: null }; - const grant = workerService.takeMintedCredential(binding); - expect(grant).toMatchObject({ - credential: CREDENTIAL, - ownerEpoch: 1, - sessionId: null, - }); - expect(workerService.acknowledgeCredentialDelivery(grant!)).toBe(true); - expect(store.getCredential(result.environmentId)).toMatchObject({ deliveredAtMs: nowMs }); - expect(workerService.takeMintedCredential(binding)).toBeUndefined(); - }); - - it("creates a nested environment from its parent's snapshot after config drift", async () => { - const provisionedProfiles: WorkerProfile[] = []; - let lease = 0; - let credential = 0; - const workerService = createService( - createProvider({ - provision: async (profile) => { - provisionedProfiles.push(structuredClone(profile)); - lease += 1; - return { leaseId: `lease-${lease}`, ssh: SSH_ENDPOINT }; - }, - }), - { - generateWorkerCredential: () => `nested-worker-credential-${(credential += 1)}`, - }, - ); - const parent = await workerService.create("development", "parent-profile-snapshot"); - getDevelopmentProfile().settings = { region: "mutated" }; - - const child = await workerService.createFromProfileSnapshot( - { - profileId: parent.profileId, - providerId: parent.providerId, - profileSnapshot: parent.profileSnapshot, - }, - "child-profile-snapshot", - ); - - expect(provisionedProfiles).toEqual([{ region: "test" }, { region: "test" }]); - expect(child).toMatchObject({ - profileId: parent.profileId, - providerId: parent.providerId, - profileSnapshot: parent.profileSnapshot, - }); - }); - - it("adopts a matching milestone-1 row that predates worker credentials", async () => { - const environmentId = "worker-milestone-one"; - seedReady(environmentId); - database.db - .prepare("DELETE FROM worker_environment_credentials WHERE environment_id = ?") - .run(environmentId); - database.db - .prepare("UPDATE worker_environments SET owner_epoch = 0 WHERE environment_id = ?") - .run(environmentId); - const workerService = createService( - createProvider({ - inspect: async () => { - throw new Error("provider unavailable"); - }, - }), - ); - - await workerService.reconcileOnce(); - - expect(store.get(environmentId)?.ownerEpoch).toBe(1); - expect(store.getCredential(environmentId)).toMatchObject({ ownerEpoch: 1, sessionId: null }); - expect( - workerService.takeMintedCredential({ environmentId, ownerEpoch: 1, sessionId: null }), - ).toMatchObject({ - credential: CREDENTIAL, - ownerEpoch: 1, - }); - expect(store.get(environmentId)?.lastError).toBe("provider unavailable"); - expect(bootstrapWorker).not.toHaveBeenCalled(); - }); - - it("admits an npm-installed worker from canonical bundle identity without registry access", async () => { - const environmentId = "worker-npm-admission"; - seedReady(environmentId, "npm"); - prepareInstallation = vi.fn(async (install) => { - if (install === "npm") { - throw new Error("registry unavailable"); - } - return BUNDLE_ARTIFACT; - }); - const workerService = createService(createProvider()); - - await expect(workerService.admitWorker(admissionFor(environmentId))).resolves.toMatchObject({ - ok: true, - }); - expect(prepareInstallation).toHaveBeenCalledTimes(1); - expect(prepareInstallation).toHaveBeenCalledWith("bundle"); - }); - - it("fences transcript commits by current epoch and exact session credential binding", async () => { - const environmentId = "worker-transcript-fence"; - const sessionId = "session-transcript-fence"; - const identity = seedAttachedIdentity(environmentId, sessionId); - const applyTranscriptCommit = successfulTranscriptCommit("entry-1"); - const workerService = createService(createProvider(), { applyTranscriptCommit }); - const request = transcriptRequest(identity, "hello"); - - await expect(workerService.commitTranscript(identity, request)).resolves.toMatchObject({ - ok: true, - }); - expect(applyTranscriptCommit).toHaveBeenCalledOnce(); - - await expect( - workerService.commitTranscript(identity, { - ...request, - runEpoch: identity.ownerEpoch + 1, - seq: 2, - }), - ).resolves.toEqual({ ok: false, reason: "epoch-mismatch" }); - database.db - .prepare("UPDATE worker_environment_credentials SET session_id = ? WHERE environment_id = ?") - .run("session-other", environmentId); - await expect(workerService.commitTranscript(identity, { ...request, seq: 2 })).resolves.toEqual( - { ok: false, reason: "session-not-attached" }, - ); - expect(applyTranscriptCommit).toHaveBeenCalledOnce(); - }); - - it("admits only a gateway-preclaimed worker placement and fences later requests", async () => { - const environmentId = "worker-placement-fence"; - const sessionId = "session-placement-fence"; - const { identity, placementStore, workerService } = placementHarness(environmentId, sessionId); - const admission = { - environmentId, - credential: [CREDENTIAL, environmentId, sessionId].join("-"), - sessionId, - runId: "run-1", - ownerEpoch: identity.ownerEpoch, - rpcSetVersion: 1, - handshake: BOOTSTRAP_RECEIPT, - }; - - await expect(workerService.admitWorker(admission)).resolves.toMatchObject({ ok: true }); - await expect(workerService.admitWorker(admission)).resolves.toMatchObject({ ok: true }); - expect(placementStore.validateWorkerTurn).toHaveBeenLastCalledWith({ - sessionId, - environmentId, - ownerEpoch: identity.ownerEpoch, - runId: "run-1", - }); - expect(placementStore.validateWorkerTurn).toHaveBeenCalledTimes(2); - expect(workerService.validateWorkerConnection(identity)).toBeNull(); - - const warmEnvironmentId = "worker-placement-warm"; - seedReady(warmEnvironmentId); - const warmAdmission = await workerService.admitWorker(admissionFor(warmEnvironmentId)); - expect(warmAdmission).toMatchObject({ ok: true }); - if (!warmAdmission.ok) { - throw new Error("warm worker admission failed"); - } - expect(workerService.validateWorkerConnection(warmAdmission.identity)).toBeNull(); - expect(placementStore.validateWorkerTurn).toHaveBeenCalledTimes(3); - - placementStore.validateWorkerTurn.mockReturnValue(false); - await expect( - workerService.admitWorker({ ...admission, runId: "run-conflict" }), - ).resolves.toEqual({ ok: false, reason: "placement-mismatch" }); - - placementStore.validateWorkerTurn.mockReturnValue(true); - nowMs += 10_000; - expect(workerService.validateWorkerConnection(identity)).toBeNull(); - expect(workerService.validateWorkerConnection(warmAdmission.identity)).toBe( - "credential-expired", - ); - await expect(workerService.admitWorker(admission)).resolves.toMatchObject({ - ok: true, - identity: { sessionId, runId: "run-1" }, - }); - - placementStore.validateWorkerTurn.mockReturnValue(false); - expect(workerService.validateWorkerConnection(identity)).toBe("placement-mismatch"); - vi.mocked(prepareInstallation).mockClear(); - await expect(workerService.admitWorker(admission)).resolves.toEqual({ - ok: false, - reason: "credential-expired", - }); - expect(prepareInstallation).not.toHaveBeenCalled(); - await expect( - workerService.commitTranscript(identity, transcriptRequest(identity, "fenced")), - ).resolves.toEqual({ ok: false, closeReason: "placement-mismatch" }); - }); - - it("does not rotate an expired delivered credential while its durable turn is active", async () => { - const environmentId = "worker-expired-active-turn"; - const sessionId = "session-expired-active-turn"; - const liveEvents = createLiveEvents(); - const { identity, workerService } = placementHarness(environmentId, sessionId, { - liveEvents, - }); - store.markCredentialDelivered({ - environmentId, - credentialHash: identity.credentialHash, - ownerEpoch: identity.ownerEpoch, - sessionId, - deliveredAtMs: nowMs, - }); - nowMs = identity.credentialExpiresAtMs; - - await workerService.reconcileOnce(); - - expect(store.getCredential(environmentId)?.credentialHash).toBe(identity.credentialHash); - expect(liveEvents.rotateCredential).not.toHaveBeenCalled(); - }); - - it("persists worker transcript and terminal live ACK cursors", async () => { - const applyTranscriptCommit = successfulTranscriptCommit("entry-placement"); - const { liveEvents } = sequencedLiveEvents(); - const { identity, placementStore, workerService } = placementHarness( - "worker-placement-ack", - "session-placement-ack", - { - applyTranscriptCommit, - liveEvents, - }, - ); - const binding = placementBinding(identity); - - await expect( - workerService.commitTranscript(identity, transcriptRequest(identity, "commit", { seq: 7 })), - ).resolves.toMatchObject({ ok: true }); - expect(placementStore.updateAckCursors).toHaveBeenCalledWith({ - ...binding, - transcriptSeq: 7, - }); - - await expect(workerService.pushLiveEvent(identity, terminalEvent(identity))).resolves.toEqual({ - ok: true, - result: { ackedSeq: 1 }, - }); - expect(placementStore.updateAckCursors).toHaveBeenLastCalledWith({ - ...binding, - liveSeq: 1, - workspaceResultPending: true, - }); - }); - - it("uses worker finishing as the durable workspace-result fence", async () => { - const { liveEvents } = sequencedLiveEvents(); - const { identity, placementStore, workerService } = placementHarness( - "worker-placement-finishing", - "session-placement-finishing", - { liveEvents }, - ); - const terminal = terminalEvent(identity); - const finishing = { - ...terminal, - event: { - kind: "lifecycle" as const, - payload: { phase: "finishing" as const, startedAt: 1, endedAt: 2 }, - }, - }; - - await expect(workerService.pushLiveEvent(identity, finishing)).resolves.toEqual({ - ok: true, - result: { ackedSeq: 1 }, - }); - expect(placementStore.updateAckCursors).toHaveBeenLastCalledWith({ - ...placementBinding(identity), - liveSeq: 1, - workspaceResultPending: true, - }); - }); - - it("does not ACK a transcript commit after its worker claim is fenced", async () => { - let finishCommit: (() => void) | undefined; - const commitBlocked = new Promise((resolve) => { - finishCommit = resolve; - }); - const applyTranscriptCommit = successfulTranscriptCommit( - "entry-placement-race", - () => commitBlocked, - ); - const { identity, placementStore, workerService } = placementHarness( - "worker-placement-race", - "session-placement-race", - { applyTranscriptCommit }, - ); - - const commit = workerService.commitTranscript( - identity, - transcriptRequest(identity, "commit before claim fence"), - ); - await waitForFast(() => expect(applyTranscriptCommit).toHaveBeenCalledOnce()); - placementStore.validateWorkerTurn.mockReturnValue(false); - finishCommit?.(); - - await expect(commit).resolves.toEqual({ ok: false, closeReason: "placement-mismatch" }); - expect(placementStore.validateWorkerTurn).toHaveBeenCalledTimes(2); - expect(placementStore.updateAckCursors).not.toHaveBeenCalled(); - }); - - it("advances the transcript cursor when a stale-base commit consumes its sequence", async () => { - const applyTranscriptCommit = vi - .fn>() - .mockResolvedValueOnce({ ok: false, reason: "stale-base-leaf" }) - .mockResolvedValueOnce({ ok: false, reason: "invalid-batch" }); - const { identity, placementStore, workerService } = placementHarness( - "worker-placement-stale", - "session-placement-stale", - { applyTranscriptCommit }, - ); - const request = transcriptRequest(identity, "stale commit", { - seq: 11, - baseLeafId: "stale-leaf", - }); - - await expect(workerService.commitTranscript(identity, request)).resolves.toEqual({ - ok: false, - reason: "stale-base-leaf", - }); - expect(placementStore.updateAckCursors).toHaveBeenCalledWith({ - ...placementBinding(identity), - transcriptSeq: 11, - }); - - await expect( - workerService.commitTranscript(identity, { ...request, seq: 12 }), - ).resolves.toEqual({ ok: false, reason: "invalid-batch" }); - expect(placementStore.updateAckCursors).toHaveBeenCalledOnce(); - }); - - it("fences after a buffered terminal event becomes acknowledged by a gap fill", async () => { - const applyTranscriptCommit = successfulTranscriptCommit("entry-after-terminal-gap"); - const { apply: liveApply, liveEvents } = sequencedLiveEvents((seq) => (seq === 1 ? 2 : 0)); - const { identity, placementStore, workerService } = placementHarness( - "worker-placement-gap", - "session-placement-gap", - { - applyTranscriptCommit, - liveEvents, - }, - ); - - await expect( - workerService.pushLiveEvent(identity, terminalEvent(identity, { seq: 2 })), - ).resolves.toEqual({ ok: true, result: { ackedSeq: 0 } }); - expect(placementStore.updateAckCursors).toHaveBeenCalledWith({ - ...placementBinding(identity), - liveSeq: 0, - workspaceResultPending: true, - }); - - await expect( - workerService.pushLiveEvent(identity, assistantEvent(identity, "fills gap")), - ).resolves.toEqual({ ok: true, result: { ackedSeq: 2 } }); - await expect( - workerService.commitTranscript(identity, transcriptRequest(identity, "late transcript")), - ).resolves.toEqual({ ok: false, closeReason: "placement-mismatch" }); - await expect( - workerService.pushLiveEvent( - identity, - assistantEvent(identity, "late", { lastAckedSeq: 2, seq: 3 }), - ), - ).resolves.toEqual({ ok: false, closeReason: "placement-mismatch" }); - expect(applyTranscriptCommit).not.toHaveBeenCalled(); - expect(liveApply).toHaveBeenCalledTimes(2); - }); - - it("applies a terminal ACK only after its transcript commit finishes", async () => { - let finishCommit: (() => void) | undefined; - const commitBlocked = new Promise((resolve) => { - finishCommit = resolve; - }); - const applyTranscriptCommit = successfulTranscriptCommit("entry-order", () => commitBlocked); - const { liveEvents } = sequencedLiveEvents(); - const { identity, placementStore, workerService } = placementHarness( - "worker-placement-order", - "session-placement-order", - { applyTranscriptCommit, liveEvents }, - ); - - const commit = workerService.commitTranscript( - identity, - transcriptRequest(identity, "commit before terminal"), - ); - await waitForFast(() => expect(applyTranscriptCommit).toHaveBeenCalledOnce()); - const terminal = workerService.pushLiveEvent(identity, terminalEvent(identity)); - await Promise.resolve(); - expect(placementStore.updateAckCursors).not.toHaveBeenCalled(); - - finishCommit?.(); - await expect(commit).resolves.toMatchObject({ ok: true }); - await expect(terminal).resolves.toEqual({ ok: true, result: { ackedSeq: 1 } }); - expect(placementStore.updateAckCursors.mock.calls).toEqual([ - [ - { - ...placementBinding(identity), - transcriptSeq: 1, - }, - ], - [ - { - ...placementBinding(identity), - liveSeq: 1, - workspaceResultPending: true, - }, - ], - ]); - }); - - it("fences post-terminal mutations while preserving sequenced replays", async () => { - const applyTranscriptCommit = successfulTranscriptCommit("entry-terminal"); - const { apply: liveApply, liveEvents } = sequencedLiveEvents(); - const executeInference = vi.fn( - async () => ({ - type: "error", - reason: "provider-error", - message: "Provider request failed", - }), - ); - const { identity, workerService } = placementHarness( - "worker-terminal-fence", - "session-terminal-fence", - { applyTranscriptCommit, executeInference, liveEvents }, - ); - const transcript = transcriptRequest(identity, "terminal fence"); - const terminal = terminalEvent(identity); - - await expect(workerService.commitTranscript(identity, transcript)).resolves.toMatchObject({ - ok: true, - }); - await expect(workerService.pushLiveEvent(identity, terminal)).resolves.toEqual({ - ok: true, - result: { ackedSeq: 1 }, - }); - - await expect(workerService.commitTranscript(identity, transcript)).resolves.toMatchObject({ - ok: true, - }); - await expect( - workerService.commitTranscript(identity, { ...transcript, seq: 2 }), - ).resolves.toEqual({ ok: false, closeReason: "placement-mismatch" }); - expect(applyTranscriptCommit).toHaveBeenCalledTimes(2); - - await expect(workerService.pushLiveEvent(identity, terminal)).resolves.toEqual({ - ok: true, - result: { ackedSeq: 1 }, - }); - await expect( - workerService.pushLiveEvent(identity, assistantEvent(identity, "late", { seq: 2 })), - ).resolves.toEqual({ ok: false, closeReason: "placement-mismatch" }); - expect(liveApply).toHaveBeenCalledTimes(2); - - expect( - workerService.startInference(identity, inferenceRequest(identity), { - connectionId: "connection-terminal-fence", - send: vi.fn(), - }), - ).toEqual({ ok: false, closeReason: "placement-mismatch" }); - expect(workerService.cancelInference(identity, inferenceRequest(identity))).toEqual({ - ok: false, - closeReason: "placement-mismatch", - }); - expect(executeInference).not.toHaveBeenCalled(); - - const rotatedCredentialHash = hashWorkerCredential( - ["rotated", identity.environmentId, identity.sessionId].join("-"), - ); - database.db - .prepare( - "UPDATE worker_environment_credentials SET credential_hash = ? WHERE environment_id = ?", - ) - .run(rotatedCredentialHash, identity.environmentId); - const rotatedIdentity = { ...identity, credentialHash: rotatedCredentialHash }; - await expect( - workerService.commitTranscript(rotatedIdentity, { ...transcript, seq: 2 }), - ).resolves.toMatchObject({ ok: true }); - expect(applyTranscriptCommit).toHaveBeenCalledTimes(3); - }); - - it("does not treat a terminal event on an already ACKed sequence as authoritative", async () => { - const applyTranscriptCommit = successfulTranscriptCommit("entry-after-reuse"); - const { liveEvents } = sequencedLiveEvents(); - const { identity, workerService } = placementHarness( - "worker-terminal-reuse", - "session-terminal-reuse", - { applyTranscriptCommit, liveEvents }, - ); - const event = assistantEvent(identity, "first"); - - await expect(workerService.pushLiveEvent(identity, event)).resolves.toMatchObject({ ok: true }); - await expect( - workerService.pushLiveEvent(identity, terminalEvent(identity)), - ).resolves.toMatchObject({ ok: true }); - await expect( - workerService.commitTranscript(identity, transcriptRequest(identity, "still mutable")), - ).resolves.toMatchObject({ ok: true }); - expect(applyTranscriptCommit).toHaveBeenCalledOnce(); - }); - - it("fences inference by epoch and the durable session credential", async () => { - const identity = seedAttachedIdentity("worker-inference-fence", "session-inference-fence"); - const executeInference = vi.fn( - async () => ({ - type: "error", - reason: "provider-error", - message: "Provider request failed", - }), - ); - const workerService = createService(createProvider(), { executeInference }); - const request = inferenceRequest(identity); - expect( - workerService.startInference( - identity, - { ...request, sessionId: "session-other" }, - { connectionId: "connection-a", send: vi.fn() }, - ), - ).toEqual({ ok: false, reason: "session-not-attached" }); - expect( - workerService.startInference( - identity, - { ...request, runEpoch: request.runEpoch + 1 }, - { connectionId: "connection-b", send: vi.fn() }, - ), - ).toEqual({ ok: false, reason: "epoch-mismatch" }); - - const send = vi.fn(); - const started = workerService.startInference(identity, request, { - connectionId: "connection-c", - send, - }); - expect(started.ok).toBe(true); - if (!started.ok) { - throw new Error("inference fixture failed to start"); - } - database.db - .prepare( - "UPDATE worker_environment_credentials SET credential_hash = ? WHERE environment_id = ?", - ) - .run( - hashWorkerCredential(["replacement", identity.environmentId].join("-")), - identity.environmentId, - ); - started.launch(); - await waitForFast(() => expect(send).toHaveBeenCalledOnce()); - expect(executeInference).not.toHaveBeenCalled(); - expect(send.mock.calls[0]?.[0]).toMatchObject({ - event: "worker.inference.terminal", - payload: { outcome: { reason: "session-not-attached" } }, - }); - }); - - it("fences and rotates live credentials", async () => { - const environmentId = "worker-live"; - const sessionId = "session-live"; - const identity = seedAttachedIdentity(environmentId, sessionId); - const liveEvents = createLiveEvents(); - let inferenceSignal: AbortSignal | undefined; - const executeInference = vi.fn( - async ({ signal }) => { - inferenceSignal = signal; - await new Promise((resolve) => { - signal.addEventListener("abort", () => resolve(), { once: true }); - }); - return { type: "error", reason: "cancelled", message: "Inference cancelled" }; - }, - ); - const workerService = createService(createProvider(), { executeInference, liveEvents }); - const request = { ...LIVE_EVENT, runEpoch: identity.ownerEpoch }; - const push = workerService.pushLiveEvent.bind(workerService, identity); - await push(request); - await expect(push({ ...request, runEpoch: identity.ownerEpoch + 1 })).resolves.toEqual({ - ok: false, - details: { reason: "epoch-mismatch" }, - }); - const started = workerService.startInference(identity, inferenceRequest(identity), { - connectionId: "connection-rotation", - send: vi.fn(), - }); - if (!started.ok) { - throw new Error("inference fixture failed to start"); - } - started.launch(); - await waitForFast(() => expect(executeInference).toHaveBeenCalledOnce()); - database.db - .prepare("UPDATE worker_environment_credentials SET session_id = ? WHERE environment_id = ?") - .run("session-other", environmentId); - await expect(push({ ...request, seq: 2 })).resolves.toEqual({ - ok: false, - details: { reason: "session-not-attached" }, - }); - liveEvents.rotateCredential.mockClear(); - nowMs += 10_000; - await workerService.reconcileOnce(); - expect(inferenceSignal?.aborted).toBe(true); - expect(liveEvents.rotateCredential).toHaveBeenCalledWith( - expect.objectContaining({ - credentialHash: store.getCredential(environmentId)?.credentialHash, - previousCredentialHash: identity.credentialHash, - runEpoch: identity.ownerEpoch, - }), - ); - }); - - it("repairs duplicate session owners", async () => { - const sessionId = "legacy"; - const older = seedAttachedIdentity("legacy-a", sessionId); - const newer = seedAttachedIdentity("legacy-b", "other"); - database.db.exec(` - UPDATE worker_environments SET attached_session_ids_json = '["legacy"]' - WHERE environment_id = 'legacy-b'; - UPDATE worker_environment_credentials SET session_id = 'legacy' - WHERE environment_id = 'legacy-b'; - `); - - closeOpenClawStateDatabaseForTest(); - database = openOpenClawStateDatabase({ env: { OPENCLAW_STATE_DIR: root } }); - store = createWorkerEnvironmentStore({ database, now: () => nowMs }); - const liveEvents = createLiveEvents(); - const workerService = createService(createProvider(), { liveEvents }); - const event = { ...LIVE_EVENT, runEpoch: newer.ownerEpoch }; - await expect(workerService.pushLiveEvent(older, event)).resolves.toEqual({ - ok: false, - closeReason: "credential-replaced", - }); - await workerService.pushLiveEvent({ ...newer, sessionId }, event); - expect(liveEvents.apply).toHaveBeenCalledOnce(); - }); - - it("rejects attach before current bootstrap", async () => { - const staleId = "worker-stale-attach"; - const bootstrapping = seedBootstrapping(staleId); - store.transition({ - environmentId: staleId, - from: bootstrapping.state, - to: "ready", - patch: readyPatch(staleId, { ...BOOTSTRAP_RECEIPT, bundleHash: "c".repeat(64) }), - }); - const workerService = createService(createProvider()); - - await expect( - workerService.attachSession({ - environmentId: staleId, - ownerEpoch: 1, - sessionId: "session-1", - }), - ).rejects.toThrow("must bootstrap the current build"); - expect(store.get(staleId)).toMatchObject({ state: "ready", attachedSessionIds: [] }); - }); - - it("returns a bounded error when another worker owns the session", async () => { - const firstId = "worker-session-owner"; - const secondId = "worker-session-contender"; - seedReady(firstId); - seedReady(secondId); - const workerService = createService(createProvider()); - - await workerService.attachSession({ - environmentId: firstId, - ownerEpoch: 1, - sessionId: "session-owned", - }); - await expect( - workerService.attachSession({ - environmentId: secondId, - ownerEpoch: 1, - sessionId: "session-owned", - }), - ).rejects.toMatchObject({ - code: "invalid_state", - message: - "Session session-owned is already attached to worker environment worker-session-owner", - }); - expect(store.get(secondId)).toMatchObject({ state: "ready", attachedSessionIds: [] }); - }); - - it("requires session reclaim before operator destruction of an attached worker", async () => { - const environmentId = "worker-session-reclaim"; - seedReady(environmentId); - const workerService = createService(createProvider()); - await workerService.attachSession({ - environmentId, - ownerEpoch: 1, - sessionId: "session-reclaim", - }); - - await expect(workerService.destroyUnattached(environmentId)).rejects.toMatchObject({ - code: "invalid_state", - message: "Attached cloud workers must be stopped through sessions.reclaim", - }); - expect(store.get(environmentId)).toMatchObject({ - state: "attached", - attachedSessionIds: ["session-reclaim"], - }); - }); - - it("stops the tunnel after live binding rollback", async () => { - const environmentId = "live-bind-fail"; - seedReady(environmentId); - const liveEvents = createLiveEvents({ - bindSession: vi.fn(() => { - throw new Error("bind failed"); - }), - }); - const tunnelManager = { - stop: vi.fn(async () => {}), - stopAll: vi.fn(async () => {}), - } as unknown as WorkerTunnelManager; - const workerService = createService(createProvider(), { liveEvents, tunnelManager }); - - await expect( - workerService.attachSession({ environmentId, ownerEpoch: 1, sessionId: "session-live" }), - ).rejects.toThrow("Attached session target is unavailable"); - expect(tunnelManager.stop).toHaveBeenCalledWith(environmentId, 1); - expect(store.get(environmentId)).toMatchObject({ state: "idle", attachedSessionIds: [] }); - }); - - it("renews in place and binds delivery acknowledgement to the exact grant", async () => { - const environmentId = "worker-credential-replacement"; - seedReady(environmentId); - let credentialSequence = 0; - const workerService = createService(createProvider(), { - generateWorkerCredential: () => [CREDENTIAL, String(++credentialSequence)].join("-"), - workerCredentialTtlMs: 100, - }); - - const binding = { environmentId, ownerEpoch: 1, sessionId: null }; - await workerService.reconcileOnce(); - const previous = workerService.takeMintedCredential(binding)!; - nowMs += 100; - expect(workerService.takeMintedCredential(binding)).toBeUndefined(); - await workerService.reconcileOnce(); - const renewal = workerService.takeMintedCredential(binding)!; - expect(renewal).toMatchObject({ ownerEpoch: 1, sessionId: null }); - expect(store.get(environmentId)?.ownerEpoch).toBe(1); - expect(workerService.acknowledgeCredentialDelivery(previous)).toBe(false); - expect(workerService.takeMintedCredential(binding)).toMatchObject({ - deliveryId: renewal.deliveryId, - }); - expect(workerService.acknowledgeCredentialDelivery(renewal)).toBe(true); - }); - - it("recovers an undelivered atomic session credential after restart without changing owner", async () => { - const environmentId = "worker-attach-restart"; - seedReady(environmentId); - let credentialSequence = 0; - const stopTunnel = vi.fn(async () => { - throw new Error("tunnel stop interrupted"); - }); - const tunnelManager = { - start: vi.fn(), - stop: stopTunnel, - stopAll: vi.fn(async () => {}), - status: () => "stopped" as const, - desktop: { - acquire: vi.fn(), - attachObserver: vi.fn(), - stop: vi.fn(async () => {}), - stopAll: vi.fn(async () => {}), - }, - } as unknown as WorkerTunnelManager; - const options = { - generateWorkerCredential: () => [CREDENTIAL, String(++credentialSequence)].join("-"), - tunnelManager, - }; - const first = createService(createProvider(), options); - await first.reconcileOnce(); - await expect( - first.attachSession({ environmentId, ownerEpoch: 1, sessionId: "session-1" }), - ).rejects.toThrow("tunnel stop interrupted"); - const binding = { environmentId, ownerEpoch: 2, sessionId: "session-1" }; - const lostHash = store.getCredential(environmentId)?.credentialHash; - - expect(stopTunnel).toHaveBeenCalledWith(environmentId, 1); - expect(store.get(environmentId)).toMatchObject({ state: "attached", ownerEpoch: 2 }); - expect(first.takeMintedCredential(binding)).toBeUndefined(); - - await first.stop(); - const restarted = createService(createProvider(), options); - await restarted.reconcileOnce(); - - const recovered = restarted.takeMintedCredential(binding); - expect(recovered?.deliveryId).not.toBe(lostHash); - expect(restarted.acknowledgeCredentialDelivery(recovered!)).toBe(true); - const deliveredHash = store.getCredential(environmentId)?.credentialHash; - - await restarted.stop(); - const deliveredRestart = createService(createProvider(), options); - await deliveredRestart.reconcileOnce(); - expect(deliveredRestart.takeMintedCredential(binding)).toBeUndefined(); - expect(store.getCredential(environmentId)?.credentialHash).toBe(deliveredHash); - }); - - it("stays bootstrapping until the SSH install receipt is durable", async () => { - let finishBootstrap: (() => void) | undefined; - const bootstrapPending = new Promise((resolve) => { - finishBootstrap = resolve; - }); - bootstrapWorker = vi.fn(async () => { - await bootstrapPending; - return BOOTSTRAP_RECEIPT; - }); - const creation = createService(createProvider()).create("development", "request-bootstrap"); - - await waitForFast(() => - expect(store.list()[0]).toMatchObject({ - state: "bootstrapping", - bootstrapReceipt: null, - }), - ); - finishBootstrap?.(); - - await expect(creation).resolves.toMatchObject({ - state: "ready", - bootstrapReceipt: BOOTSTRAP_RECEIPT, - }); - }); - - it("records installation preparation failure before allocating a lease", async () => { - prepareInstallation = vi.fn(async () => { - throw new Error("npm install requires a released gateway package"); - }); - const provision = vi.fn(createProvider().provision); - const workerService = createService(createProvider({ provision })); - - await expect( - workerService.create("development", "request-preparation-failure"), - ).rejects.toMatchObject({ - code: "bootstrap_failure", - message: expect.stringContaining("npm install requires a released gateway package"), - } satisfies Partial); - - expect(provision).not.toHaveBeenCalled(); - expect(store.list()[0]).toMatchObject({ - state: "failed", - leaseId: null, - lastError: "npm install requires a released gateway package", - }); - expect(workerService.list()[0]).toMatchObject({ - state: "failed", - error: "npm install requires a released gateway package", - }); - }); - - it("keeps a remotely bootstrapped lease retryable when receipt persistence fails", async () => { - const durableStore = store; - let persistenceFails = true; - store = { - ...store, - transition(input) { - if (persistenceFails && input.from === "bootstrapping" && input.to === "ready") { - persistenceFails = false; - throw new Error("receipt database write failed"); - } - return durableStore.transition(input); - }, - }; - const destroy = vi.fn(async () => {}); - const workerService = createService(createProvider({ destroy })); - - await expect( - workerService.create("development", "request-receipt-write-failure"), - ).rejects.toThrow("receipt database write failed"); - expect(store.list()[0]).toMatchObject({ state: "bootstrapping", leaseId: "lease-1" }); - expect(destroy).not.toHaveBeenCalled(); - - await workerService.reconcileOnce(); - expect(store.list()[0]).toMatchObject({ state: "ready", bootstrapReceipt: BOOTSTRAP_RECEIPT }); - expect(bootstrapWorker).toHaveBeenCalledTimes(2); - }); - - it("tears down the lease and records a bounded bootstrap failure", async () => { - // Assembled at runtime so review-bundle secret scanners do not flag a key-shaped literal. - const secret = ["sk", "proj", "bootstrap", "abcdefghijklmnopqrstuvwxyz"].join("-"); - bootstrapWorker = vi.fn(async () => { - throw new Error(`remote bootstrap rejected ${secret}`); - }); - const destroy = vi.fn(async () => {}); - const workerService = createService(createProvider({ destroy })); - - const creation = workerService.create("development", "request-bootstrap-failure"); - await expect(creation).rejects.toMatchObject({ - code: "bootstrap_failure", - message: expect.stringContaining("Worker bootstrap failed: remote bootstrap rejected"), - } satisfies Partial); - await expect(creation).rejects.not.toThrow(secret); - - expect(destroy).toHaveBeenCalledTimes(1); - expect(store.list()[0]).toMatchObject({ - state: "failed", - leaseId: null, - sshEndpoint: null, - bootstrapReceipt: null, - lastError: expect.stringContaining("remote bootstrap rejected"), - }); - expect(store.list()[0]?.lastError).not.toContain(secret); - }); - - it("projects bounded bootstrap detail through sessions.describe after failed dispatch", async () => { - // Assembled at runtime so review-bundle secret scanners do not flag a key-shaped literal. - const secret = ["sk", "proj", "placement", "abcdefghijklmnopqrstuvwxyz"].join("-"); - bootstrapWorker = vi.fn(async () => { - throw new Error(`remote bootstrap rejected ${secret} ${"failure ".repeat(200)}`); - }); - const workerService = createService(createProvider()); - const placements = createWorkerSessionPlacementStore({ database, now: () => nowMs }); - const dispatch = createWorkerPlacementDispatchService({ - placements, - environments: workerService, - workspaceOperations: createWorkerWorkspaceOperationCoordinator(), - runLocalBarrier: async ({ startDispatch }) => startDispatch(), - runActivationBarrier: async ({ activate }) => activate(), - runReclaimBarrier: async ({ reclaim }) => await reclaim("/gateway/workspace"), - resolveWorkspacePath: async () => "/gateway/workspace", - reportWorkspaceResultConflict: async () => {}, - resolveWorkspaceResultConflict: async () => undefined, - }); - - await expect( - dispatch.dispatch({ - sessionId: "session-bootstrap-failure", - sessionKey: "agent:main:session-bootstrap-failure", - agentId: "main", - profileId: "development", - }), - ).rejects.toThrow("Worker bootstrap failed: remote bootstrap rejected"); - - const persisted = expectDefined( - placements.get("session-bootstrap-failure"), - "failed worker placement", - ); - const sessionStorePath = path.join(root, "sessions.json"); - await writeSessionStore({ - entries: { main: { sessionId: persisted.sessionId, updatedAt: nowMs } }, - storePath: sessionStorePath, - }); - const described = await directSessionReq<{ session: GatewaySessionRow | null }>( - "sessions.describe", - { key: "main" }, - { - context: { - getRuntimeConfig: () => ({ session: { store: sessionStorePath } }), - workerSessionPlacementService: placements, - }, - }, - ); - const describedPlacement = described.payload?.session?.placement; - expect(described).toMatchObject({ ok: true }); - expect(describedPlacement).toMatchObject({ - state: "failed", - recoveryError: expect.stringContaining("remote bootstrap rejected"), - }); - if (describedPlacement?.state !== "failed") { - throw new Error("sessions.describe did not project the failed worker placement"); - } - expect(describedPlacement.recoveryError).not.toContain(secret); - expect(describedPlacement.recoveryError.length).toBeLessThanOrEqual(1_024); - }); - - it("keeps an indeterminate bootstrap teardown retryable", async () => { - bootstrapWorker = vi.fn(async () => { - throw new Error("remote bootstrap failed"); - }); - let teardownFails = true; - const workerService = createService( - createProvider({ - destroy: async () => { - if (teardownFails) { - throw new Error("provider teardown timed out"); - } - }, - }), - ); - - await expect( - workerService.create("development", "request-bootstrap-cleanup"), - ).rejects.toMatchObject({ - code: "bootstrap_failure", - message: "Worker bootstrap failed; teardown is pending: remote bootstrap failed", - } satisfies Partial); - expect(store.list()[0]).toMatchObject({ - state: "destroying", - leaseId: "lease-1", - destroyRequestedAtMs: expect.any(Number), - teardownTerminalState: "failed", - lastError: "remote bootstrap failed", - }); - - teardownFails = false; - await workerService.reconcileOnce(); - expect(store.list()[0]).toMatchObject({ - state: "failed", - leaseId: null, - sshEndpoint: null, - lastError: expect.stringContaining("remote bootstrap failed"), - }); - }); - - it("bounds worker identity resolution as a provider operation", async () => { - const events: string[] = []; - let finishIdentity: (() => void) | undefined; - const identityPending = new Promise((resolve) => { - finishIdentity = resolve; - }); - bootstrapWorker = vi.fn(async ({ installation, resolveIdentity, signal }) => { - signal.addEventListener("abort", () => void events.push("abort"), { once: true }); - await resolveIdentity(SSH_ENDPOINT.keyRef); - return { - bundleHash: installation.bundleHash, - openclawVersion: installation.openclawVersion, - protocolFeatures: [...installation.protocolFeatures], - }; - }); - const destroy = vi.fn(async () => { - events.push("destroy"); - }); - const workerService = createService(createProvider({ destroy }), { - providerCallTimeoutMs: 5, - resolveSshIdentity: async () => { - events.push("identity:start"); - await identityPending; - events.push("identity:end"); - return { kind: "path", path: "/keys/worker" }; - }, - }); - - const creation = workerService.create("development", "request-identity-timeout"); - const creationResult = expect(creation).rejects.toMatchObject({ - code: "bootstrap_failure", - } satisfies Partial); - try { - await waitForFast(() => expect(store.list()[0]).toMatchObject({ state: "destroying" })); - expect(events).toEqual(["identity:start", "abort"]); - expect(destroy).not.toHaveBeenCalled(); - } finally { - finishIdentity?.(); - } - - await creationResult; - expect(destroy).toHaveBeenCalledOnce(); - expect(events).toEqual(["identity:start", "abort", "identity:end", "destroy"]); - expect(store.list()[0]).toMatchObject({ state: "failed", leaseId: null }); - }); - - it("aborts a timed-out SSH bootstrap before tearing down its lease", async () => { - const events: string[] = []; - bootstrapWorker = vi.fn( - async ({ signal }) => - await new Promise((_resolve, reject) => { - signal.addEventListener( - "abort", - () => { - events.push("abort"); - reject(new Error("SSH bootstrap aborted")); - }, - { once: true }, - ); - }), - ); - const destroy = vi.fn(async () => { - events.push("destroy"); - }); - const workerService = createService(createProvider({ destroy }), { - bootstrapCallTimeoutMs: 10, - }); - - await expect( - workerService.create("development", "request-bootstrap-timeout"), - ).rejects.toMatchObject({ - code: "bootstrap_failure", - } satisfies Partial); - - expect(events).toEqual(["abort", "destroy"]); - expect(store.list()[0]).toMatchObject({ state: "failed", leaseId: null }); - }); - - it("adopts one committed provision across a service and store restart", async () => { - const physicalLeases = new Set(); - const operationIds: string[] = []; - const destroyed: string[] = []; - let creates = 0; - let loseFirstReply = true; - const provider = () => - createProvider({ - provision: async (_profile, operationId) => { - operationIds.push(operationId); - if (!physicalLeases.has("lease-restarted")) { - creates += 1; - physicalLeases.add("lease-restarted"); - } - if (loseFirstReply) { - loseFirstReply = false; - throw new Error("provider response was lost after commit"); - } - return { leaseId: "lease-restarted", ssh: SSH_ENDPOINT }; - }, - destroy: async ({ leaseId }) => { - destroyed.push(leaseId); - physicalLeases.delete(leaseId); - }, - }); - const first = createService(provider()); - - await expect(first.create("development", "request-restart-replay")).rejects.toMatchObject({ - code: "provider_failure", - } satisfies Partial); - const environmentId = expectDefined( - store.list()[0], - "persisted provision intent", - ).environmentId; - const operationId = expectDefined( - store.get(environmentId), - "persisted provision record", - ).provisionOperationId; - expect(operationId).toMatch(/^provision:v2:[a-f0-9]{64}$/u); - expect(store.get(environmentId)).toMatchObject({ state: "provisioning", leaseId: null }); - - await first.stop(); - service = undefined; - closeOpenClawStateDatabaseForTest(); - database = openOpenClawStateDatabase({ env: { OPENCLAW_STATE_DIR: root } }); - store = createWorkerEnvironmentStore({ database, now: () => nowMs }); - - const restarted = createService(provider()); - restarted.start(); - await waitForFast(() => - expect(store.get(environmentId)).toMatchObject({ - state: "ready", - leaseId: "lease-restarted", - lastError: null, - }), - ); - await restarted.destroy(environmentId); - - expect(creates).toBe(1); - expect(operationIds).toEqual([operationId, operationId]); - expect(destroyed).toEqual(["lease-restarted"]); - expect(physicalLeases.size).toBe(0); - expect(store.get(environmentId)).toMatchObject({ - state: "destroyed", - leaseId: "lease-restarted", - }); - }); - - it("records a permanent legacy provision replay failure without allocating", async () => { - const legacyOperationId = `provision:${"0".repeat(64)}`; - const intent = store.createIntent({ - environmentId: "worker-legacy-provision", - providerId: "fake", - profileId: "development", - profileSnapshot: { settings: { region: "test" } }, - provisionOperationId: legacyOperationId, - }); - store.transition({ - environmentId: intent.environmentId, - from: intent.state, - to: "provisioning", - }); - const allocate = vi.fn(async () => ({ leaseId: "must-not-exist", ssh: SSH_ENDPOINT })); - const provider = createProvider({ - provision: async (_profile, operationId) => { - if (operationId === legacyOperationId) { - throw new WorkerProviderError("Legacy Crabbox provision state cannot be replayed safely"); - } - return await allocate(); - }, - }); - - await createService(provider).reconcileOnce(); - - expect(allocate).not.toHaveBeenCalled(); - expect(store.get(intent.environmentId)).toMatchObject({ - state: "failed", - leaseId: null, - lastError: "Legacy Crabbox provision state cannot be replayed safely", - }); - }); - - it("does not resolve a provider provision timeout when the service override is set", async () => { - const resolveProvisionTimeoutMs = vi.fn(() => { - throw new Error("provider timeout hook must not run"); - }); - const workerService = createService(createProvider({ resolveProvisionTimeoutMs }), { - providerCallTimeoutMs: 1_000, - }); - - await expect( - workerService.create("development", "request-provider-timeout-override"), - ).resolves.toMatchObject({ state: "ready" }); - expect(resolveProvisionTimeoutMs).not.toHaveBeenCalled(); - }); - - it.each([ - ["zero", 0], - ["negative", -1], - ["fractional", 1.5], - ["non-finite", Number.NaN], - ["timer overflow", MAX_TIMER_TIMEOUT_MS + 1], - ])("rejects a %s provider provision timeout before allocation", async (_label, timeoutMs) => { - const provision = vi.fn(async () => ({ leaseId: "lease-invalid-timeout", ssh: SSH_ENDPOINT })); - const workerService = createService( - createProvider({ - provision, - resolveProvisionTimeoutMs: () => timeoutMs, - }), - ); - - await expect( - workerService.create("development", `request-invalid-provider-timeout-${String(timeoutMs)}`), - ).rejects.toMatchObject({ - code: "invalid_profile", - message: expect.stringContaining("Worker provider provision timeout must be an integer"), - } satisfies Partial); - expect(provision).not.toHaveBeenCalled(); - }); - - it("serializes destroy and provision replay behind a timed-out provider operation", async () => { - const events: string[] = []; - const operationIds: string[] = []; - let active = 0; - let maxActive = 0; - let originalProvisionCalls = 0; - let finishFirstProvision: (() => void) | undefined; - const firstProvisionPending = new Promise((resolve) => { - finishFirstProvision = resolve; - }); - const destroy = vi.fn(async () => { - events.push("destroy:start"); - active += 1; - maxActive = Math.max(maxActive, active); - active -= 1; - events.push("destroy:end"); - }); - const provider = createProvider({ - provision: async (_profile, operationId) => { - originalProvisionCalls += 1; - const call = originalProvisionCalls; - operationIds.push(operationId); - events.push(`provision:${call}:start`); - active += 1; - maxActive = Math.max(maxActive, active); - if (call === 1) { - await firstProvisionPending; - } - active -= 1; - events.push(`provision:${call}:end`); - return { leaseId: "lease-timeout-replay", ssh: SSH_ENDPOINT }; - }, - destroy, - resolveProvisionTimeoutMs: () => 20, - }); - const workerService = createService(provider); - const creation = workerService.create("development", "request-provider-timeout-race"); - const creationResult = expect(creation).rejects.toMatchObject({ - code: "provider_failure", - } satisfies Partial); - let environmentId: string | undefined; - let teardownResult: Promise | undefined; - try { - await waitForFast(() => expect(events).toEqual(["provision:1:start"])); - const queuedEnvironmentId = expectDefined( - store.list()[0], - "timed-out provision row", - ).environmentId; - environmentId = queuedEnvironmentId; - const teardown = workerService.destroy(queuedEnvironmentId); - teardownResult = expect(teardown).resolves.toMatchObject({ state: "destroyed" }); - await creationResult; - await waitForFast(() => - expect(store.get(queuedEnvironmentId)?.destroyRequestedAtMs).not.toBeNull(), - ); - expect(originalProvisionCalls).toBe(1); - expect(destroy).not.toHaveBeenCalled(); - expect(maxActive).toBe(1); - } finally { - finishFirstProvision?.(); - } - - await teardownResult; - const finalEnvironmentId = expectDefined(environmentId, "timed-out provision environment id"); - expect(operationIds).toHaveLength(2); - expect(new Set(operationIds).size).toBe(1); - expect(maxActive).toBe(1); - expect(events).toEqual([ - "provision:1:start", - "provision:1:end", - "provision:2:start", - "provision:2:end", - "destroy:start", - "destroy:end", - ]); - expect(store.get(finalEnvironmentId)).toMatchObject({ state: "destroyed" }); - }); - - it("adopts an indeterminate allocation before a replay preparation failure", async () => { - const events: string[] = []; - let preparationFails = false; - prepareInstallation = vi.fn(async () => { - events.push("prepare"); - if (preparationFails) { - throw new Error("persisted bundle is unavailable"); - } - return BUNDLE_ARTIFACT; - }); - let provisionCalls = 0; - const operationIds: string[] = []; - const provider = createProvider({ - provision: async (_profile, operationId) => { - events.push("provision"); - provisionCalls += 1; - operationIds.push(operationId); - if (provisionCalls === 1) { - throw new Error("provision response was lost"); - } - return { leaseId: "lease-replayed", ssh: SSH_ENDPOINT }; - }, - destroy: async () => void events.push("destroy"), - }); - const workerService = createService(provider); - - await expect( - workerService.create("development", "request-lost-provision"), - ).rejects.toMatchObject({ - code: "provider_failure", - } satisfies Partial); - preparationFails = true; - await workerService.reconcileOnce(); - - expect(events).toEqual(["prepare", "provision", "provision", "prepare", "destroy"]); - expect(new Set(operationIds).size).toBe(1); - expect(store.list()[0]).toMatchObject({ - state: "failed", - leaseId: null, - sshEndpoint: null, - teardownTerminalState: "failed", - lastError: "persisted bundle is unavailable", - }); - }); - - it.each([ - ["missing result", null, "invalid provision result"], - [ - "malformed SSH endpoint", - { leaseId: "lease-invalid", ssh: { ...SSH_ENDPOINT, keyRef: "not-a-secret-ref" } }, - "SSH key must be a canonical SecretRef", - ], - [ - "excessive SSH fallback ports", - { - leaseId: "lease-invalid", - ssh: { - ...SSH_ENDPOINT, - fallbackPorts: Array.from({ length: 11 }, (_, index) => 2300 + index), - }, - }, - "SSH fallback ports cannot exceed 10", - ], - [ - "invalid shared-host declaration", - { leaseId: "lease-invalid", ssh: SSH_ENDPOINT, sharedHost: "yes" }, - "invalid provision result", - ], - [ - "unsupported desktop protocol", - { leaseId: "lease-invalid", ssh: SSH_ENDPOINT, desktop: { protocol: "rdp", port: 5900 } }, - 'desktop protocol must be "rfb"', - ], - [ - "invalid desktop port", - { leaseId: "lease-invalid", ssh: SSH_ENDPOINT, desktop: { protocol: "rfb", port: 0 } }, - "desktop port must be an integer", - ], - [ - "relative desktop password path", - { - leaseId: "lease-invalid", - ssh: SSH_ENDPOINT, - desktop: { protocol: "rfb", port: 5900, passwordFilePath: "vnc.password" }, - }, - "desktop password file path must be absolute", - ], - [ - "unrecognized desktop app metadata", - { - leaseId: "lease-invalid", - ssh: SSH_ENDPOINT, - desktop: { - protocol: "rfb", - port: 5900, - apps: [ - { - id: "browser", - executablePath: "/usr/local/bin/openclaw-worker-browser", - cdpPort: 9222, - command: "chromium", - }, - ], - }, - }, - "browser desktop app contains unknown fields", - ], - ])("keeps %s from a provider retryable", async (_name, result, error) => { - const workerService = createService(createProvider({ provision: async () => result as never })); - - await expect(workerService.create("development", "request-malformed")).rejects.toMatchObject({ - code: "provider_failure", - message: expect.stringContaining(error), - } satisfies Partial); - expect(store.list()[0]).toMatchObject({ - state: "provisioning", - lastError: expect.stringContaining(error), - }); - }); - - it("rejects plaintext secret fields before persisting intent", async () => { - getDevelopmentProfile().settings = { - keyRef: "not-a-secret-ref", - }; - const provision = vi.fn(createProvider().provision); - - await expect( - createService(createProvider({ provision })).create("development", "request-secret"), - ).rejects.toMatchObject({ code: "invalid_profile" }); - expect(provision).not.toHaveBeenCalled(); - expect(store.list()).toEqual([]); - }); - - it("records permanent provider profile rejection as terminal", async () => { - let provisionCalls = 0; - const provider = createProvider({ - provision: async () => { - provisionCalls += 1; - throw new WorkerProviderError("region is required"); - }, - }); - const workerService = createService(provider); - - await expect(workerService.create("development", "request-invalid")).rejects.toMatchObject({ - code: "invalid_profile", - message: expect.stringContaining("region is required"), - } satisfies Partial); - const record = expectDefined(store.list()[0], "store.list()[0] test invariant"); - expect(record).toMatchObject({ state: "failed", lastError: "region is required" }); - - await workerService.reconcileOnce(); - await expect(workerService.destroy(record.environmentId)).resolves.toMatchObject({ - state: "failed", - }); - expect(provisionCalls).toBe(1); - }); - - it("rejects non-canonical profile ids before persistence", async () => { - const workerService = createService(createProvider()); - - await expect(workerService.create(" development ", "request-spaced")).rejects.toMatchObject({ - code: "invalid_profile", - } satisfies Partial); - expect(store.list()).toEqual([]); - }); - - it.each(["direct destroy", "restart reconcile"] as const)( - "cancels a requested intent without allocating on %s", - async (mode) => { - const intent = store.createIntent({ - environmentId: `worker-cancel-${mode}`, - providerId: "fake", - profileId: "development", - profileSnapshot: { settings: { region: "test" } }, - provisionOperationId: `provision:cancel-${mode}`, - }); - const provision = vi.fn(createProvider().provision); - const workerService = createService(createProvider({ provision })); - - if (mode === "direct destroy") { - await workerService.destroy(intent.environmentId); - } else { - store.requestDestroy({ environmentId: intent.environmentId, state: "requested" }); - providersEnabled = false; - await workerService.reconcileOnce(); - } - - expect(provision).not.toHaveBeenCalled(); - expect(store.get(intent.environmentId)).toMatchObject({ - state: "failed", - lastError: "Provisioning canceled before provider allocation", - destroyRequestedAtMs: expect.any(Number), - }); - }, - ); - - it("inspects a persisted lease with its profile snapshot after profile removal", async () => { - seedBootstrapping("worker-crash"); - config.cloudWorkers!.profiles = {}; - const inspected: WorkerLifecycleLease[] = []; - const provider = createProvider({ - inspect: async (lease) => { - inspected.push(lease); - return { status: "active" }; - }, - provision: async () => { - throw new Error("provision must not run for a known lease"); - }, - }); - - await createService(provider).reconcileOnce(); - - expect(inspected).toEqual([{ leaseId: "lease:worker-crash", profile: { region: "test" } }]); - expect(store.get("worker-crash")).toMatchObject({ - state: "ready", - bootstrapReceipt: BOOTSTRAP_RECEIPT, - }); - expect(prepareInstallation).toHaveBeenCalledWith("bundle"); - expect(bootstrapWorker).toHaveBeenCalledTimes(1); - }); - - it("skips an active lease whose durable receipt matches the lifecycle bundle", async () => { - seedReady("worker-current"); - - await createService(createProvider()).reconcileOnce(); - - expect(store.get("worker-current")).toMatchObject({ - state: "ready", - bootstrapReceipt: BOOTSTRAP_RECEIPT, - }); - expect(prepareInstallation).toHaveBeenCalledWith("bundle"); - expect(bootstrapWorker).not.toHaveBeenCalled(); - }); - - it("re-enters bootstrapping when the durable receipt has a stale bundle hash", async () => { - const bootstrapping = seedBootstrapping("worker-stale"); - store.transition({ - environmentId: bootstrapping.environmentId, - from: "bootstrapping", - to: "ready", - patch: readyPatch(bootstrapping.environmentId, { - ...BOOTSTRAP_RECEIPT, - bundleHash: "b".repeat(64), - }), - }); - - await createService(createProvider()).reconcileOnce(); - - expect(store.get("worker-stale")).toMatchObject({ - state: "ready", - bootstrapReceipt: BOOTSTRAP_RECEIPT, - }); - expect(bootstrapWorker).toHaveBeenCalledTimes(1); - }); - - it("tears down an attached worker whose admitted bundle is stale", async () => { - const environmentId = "worker-attached-stale"; - seedBootstrapping(environmentId); - const ready = store.transition({ - environmentId, - from: "bootstrapping", - to: "ready", - patch: readyPatch(environmentId, { - ...BOOTSTRAP_RECEIPT, - bundleHash: "b".repeat(64), - }), - }); - store.transition({ - environmentId, - from: ready.state, - to: "attached", - patch: attachedPatch(environmentId, "session-1"), - }); - const destroy = vi.fn(async () => {}); - - await createService(createProvider({ destroy })).reconcileOnce(); - - expect(destroy).toHaveBeenCalledOnce(); - expect(destroy).toHaveBeenCalledWith({ - leaseId: `lease:${environmentId}`, - profile: { region: "test" }, - }); - expect(store.get(environmentId)).toMatchObject({ - state: "destroyed", - leaseId: `lease:${environmentId}`, - attachedSessionIds: [], - lastError: null, - }); - }); - - it("does not resolve npm while an admitted receipt matches the local bundle", async () => { - const environmentId = "worker-current-npm"; - seedReady(environmentId, "npm"); - prepareInstallation = vi.fn(async (install) => { - if (install === "bundle") { - return BUNDLE_ARTIFACT; - } - throw new Error("npm registry is unavailable"); - }); - const destroy = vi.fn(async () => {}); - const workerService = createService(createProvider({ destroy })); - - await workerService.reconcileOnce(); - - expect(prepareInstallation).toHaveBeenCalledTimes(1); - expect(prepareInstallation).toHaveBeenCalledWith("bundle"); - expect(destroy).not.toHaveBeenCalled(); - expect(store.get(environmentId)).toMatchObject({ - state: "ready", - leaseId: `lease:${environmentId}`, - bootstrapReceipt: BOOTSTRAP_RECEIPT, - lastError: null, - }); - }); - - it("keeps an admitted lease retryable when local bundle identity is unavailable", async () => { - const environmentId = "worker-current-bundle-unavailable"; - seedReady(environmentId, "npm"); - const attachedId = "worker-attached-bundle-unavailable"; - seedReady(attachedId); - store.transition({ - environmentId: attachedId, - from: "ready", - to: "attached", - patch: attachedPatch(attachedId, "session-1"), - }); - database.db - .prepare("DELETE FROM worker_environment_credentials WHERE environment_id = ?") - .run(attachedId); - prepareInstallation = vi.fn(async () => { - throw new Error("local bundle identity is unavailable"); - }); - const destroy = vi.fn(async () => {}); - const workerService = createService(createProvider({ destroy })); - - await workerService.reconcileOnce(); - - expect(destroy).not.toHaveBeenCalled(); - expect(store.get(environmentId)).toMatchObject({ - state: "ready", - leaseId: `lease:${environmentId}`, - bootstrapReceipt: BOOTSTRAP_RECEIPT, - lastError: "local bundle identity is unavailable", - }); - expect(store.getCredential(attachedId)).toBeUndefined(); - expect( - workerService.takeMintedCredential({ - environmentId: attachedId, - ownerEpoch: 2, - sessionId: "session-1", - }), - ).toBeUndefined(); - }); - - it.each(["bootstrapping", "ready", "idle"] as const)( - "tears down a persisted %s lease when mismatched npm preparation fails", - async (state) => { - const environmentId = `worker-prepare-${state}`; - const bootstrapping = seedBootstrapping(environmentId, "npm"); - if (state !== "bootstrapping") { - const ready = store.transition({ - environmentId, - from: bootstrapping.state, - to: "ready", - patch: readyPatch(environmentId, { - ...BOOTSTRAP_RECEIPT, - bundleHash: "c".repeat(64), - }), - }); - if (state === "idle") { - store.transition({ environmentId, from: ready.state, to: "idle" }); - } - } - prepareInstallation = vi.fn(async (install) => { - if (install === "bundle") { - return BUNDLE_ARTIFACT; - } - throw new Error("released npm artifact is unavailable"); - }); - const order: string[] = []; - const tunnelManager = { - status: () => "connected" as const, - start: vi.fn(), - stop: vi.fn(async () => { - order.push("tunnel-stop"); - }), - stopAll: vi.fn(async () => {}), - } as unknown as WorkerTunnelManager; - const destroy = vi.fn(async () => { - order.push("provider-destroy"); - }); - - await createService(createProvider({ destroy }), { tunnelManager }).reconcileOnce(); - - expect(order).toEqual(["tunnel-stop", "provider-destroy"]); - expect(destroy).toHaveBeenCalledWith({ - leaseId: `lease:${environmentId}`, - profile: { region: "test" }, - }); - expect(store.get(environmentId)).toMatchObject({ - state: "failed", - leaseId: null, - sshEndpoint: null, - teardownTerminalState: "failed", - lastError: "released npm artifact is unavailable", - }); - expect(prepareInstallation).toHaveBeenCalledWith("bundle"); - expect(prepareInstallation).toHaveBeenCalledWith("npm"); - expect(bootstrapWorker).not.toHaveBeenCalled(); - }, - ); - - it("retries indeterminate teardown after a reconcile preparation failure and restart", async () => { - const environmentId = "worker-prepare-teardown-retry"; - const bootstrapping = seedBootstrapping(environmentId, "npm"); - store.transition({ - environmentId, - from: bootstrapping.state, - to: "ready", - patch: readyPatch(environmentId, { - ...BOOTSTRAP_RECEIPT, - bundleHash: "c".repeat(64), - }), - }); - prepareInstallation = vi.fn(async (install) => { - if (install === "bundle") { - return BUNDLE_ARTIFACT; - } - throw new Error("released npm artifact is unavailable"); - }); - let teardownFails = true; - const destroy = vi.fn(async () => { - if (teardownFails) { - throw new Error("provider teardown timed out"); - } - }); - const provider = createProvider({ destroy }); - const workerService = createService(provider); - - await workerService.reconcileOnce(); - expect(store.get(environmentId)).toMatchObject({ - state: "destroying", - leaseId: `lease:${environmentId}`, - teardownTerminalState: "failed", - lastError: "released npm artifact is unavailable", - }); - - await workerService.stop(); - teardownFails = false; - await createService(provider).reconcileOnce(); - - expect(destroy).toHaveBeenCalledTimes(2); - expect(store.get(environmentId)).toMatchObject({ - state: "failed", - leaseId: null, - sshEndpoint: null, - teardownTerminalState: "failed", - lastError: "released npm artifact is unavailable", - }); - }); - - it("uses the snapshotted npm selection after live config changes", async () => { - getDevelopmentProfile().install = "npm"; - const provider = createProvider({ - provision: async () => { - getDevelopmentProfile().install = "bundle"; - return { leaseId: "lease-npm", ssh: SSH_ENDPOINT }; - }, - }); - - const result = await createService(provider).create("development", "request-npm"); - - expect(result).toMatchObject({ - state: "ready", - profileSnapshot: { install: "npm" }, - }); - expect(prepareInstallation).toHaveBeenCalledWith("npm"); - expect(bootstrapWorker).toHaveBeenCalledWith({ - operationId: result.provisionOperationId, - sshEndpoint: SSH_ENDPOINT, - installation: NPM_ARTIFACT, - resolveIdentity: expect.any(Function), - signal: expect.any(AbortSignal), - }); - }); - - it("orphans unknown active leases and adopts unknown expected teardown", async () => { - seedReady("worker-unknown"); - seedReady("worker-transient"); - seedReady("worker-destroyed-unknown"); - store.requestDestroy({ environmentId: "worker-destroyed-unknown", state: "ready" }); - store.transition({ - environmentId: "worker-destroyed-unknown", - from: "ready", - to: "draining", - }); - store.transition({ - environmentId: "worker-destroyed-unknown", - from: "draining", - to: "destroying", - }); - const provider = createProvider({ - inspect: async ({ leaseId }) => { - if (leaseId !== "lease:worker-transient") { - return { status: "unknown" }; - } - throw new Error("provider temporarily unavailable"); - }, - }); - const failedTunnelStops = new Set(); - const tunnelManager = { - start: vi.fn(), - stop: vi.fn(async (environmentId: string) => { - if (!failedTunnelStops.has(environmentId)) { - failedTunnelStops.add(environmentId); - throw new Error("tunnel stop interrupted"); - } - }), - stopAll: vi.fn(async () => {}), - status: () => "connected" as const, - } as unknown as WorkerTunnelManager; - const workerService = createService(provider, { tunnelManager }); - const admitted = await workerService.admitWorker(admissionFor("worker-unknown")); - if (!admitted.ok) { - throw new Error("fixture worker admission failed"); - } - - await workerService.reconcileOnce(); - - expect(store.get("worker-unknown")?.state).toBe("draining"); - expect(store.get("worker-destroyed-unknown")?.state).toBe("destroying"); - expect(workerService.validateWorkerConnection(admitted.identity)).toBe("credential-replaced"); - expect(store.get("worker-transient")).toMatchObject({ - state: "ready", - lastError: "provider temporarily unavailable", - }); - await workerService.reconcileOnce(); - expect(tunnelManager.stop).toHaveBeenCalledTimes(4); - expect(store.get("worker-unknown")?.state).toBe("orphaned"); - expect(store.get("worker-destroyed-unknown")).toMatchObject({ state: "destroyed" }); - }); - - it.each([ - null, - { status: "future" }, - { status: "active", sharedHost: "yes" }, - { status: "unknown", sharedHost: true }, - ])("retains retryable state for malformed inspection result %#", async (inspection) => { - seedReady("worker-malformed"); - const provider = createProvider({ inspect: async () => inspection as never }); - - await createService(provider).reconcileOnce(); - - expect(store.get("worker-malformed")).toMatchObject({ - state: "ready", - lastError: expect.stringContaining("invalid inspection"), - }); - }); - - it("records provider-proven teardown without local intent as a failure", async () => { - seedReady("worker-destroyed-ready"); - seedReady("worker-destroyed-attached"); - store.transition({ - environmentId: "worker-destroyed-attached", - from: "ready", - to: "attached", - patch: attachedPatch("worker-destroyed-attached", "session-1"), - }); - seedReady("worker-destroyed-draining"); - store.transition({ - environmentId: "worker-destroyed-draining", - from: "ready", - to: "draining", - }); - const provider = createProvider({ - inspect: async () => ({ status: "destroyed" }), - destroy: async () => { - throw new Error("destroy must not run for provider-proven teardown"); - }, - }); - - const workerService = createService(provider); - await workerService.reconcileOnce(); - - for (const environmentId of [ - "worker-destroyed-ready", - "worker-destroyed-attached", - "worker-destroyed-draining", - ]) { - expect(store.get(environmentId)).toMatchObject({ - state: "failed", - attachedSessionIds: [], - lastError: "Worker environment disappeared before teardown was requested", - }); - } - }); - - it("adopts a provider-proven bootstrap teardown as failed after restart", async () => { - const bootstrapping = seedBootstrapping("worker-bootstrap-teardown-crash"); - const requested = store.requestDestroy({ - environmentId: bootstrapping.environmentId, - state: bootstrapping.state, - terminalState: "failed", - lastError: "remote bootstrap failed", - }); - const draining = store.transition({ - environmentId: requested.environmentId, - from: requested.state, - to: "draining", - patch: { lastError: requested.lastError }, - }); - store.transition({ - environmentId: draining.environmentId, - from: draining.state, - to: "destroying", - patch: { lastError: draining.lastError }, - }); - const destroy = vi.fn(async () => {}); - const provider = createProvider({ - inspect: async () => ({ status: "destroyed" }), - destroy, - }); - providersEnabled = false; - const workerService = createService(provider); - - await workerService.reconcileOnce(); - expect(store.get(bootstrapping.environmentId)).toMatchObject({ - state: "destroying", - lastError: "remote bootstrap failed", - }); - - providersEnabled = true; - await workerService.reconcileOnce(); - - expect(destroy).not.toHaveBeenCalled(); - expect(store.get(bootstrapping.environmentId)).toMatchObject({ - state: "failed", - leaseId: null, - sshEndpoint: null, - teardownTerminalState: "failed", - lastError: "remote bootstrap failed", - }); - }); - - it("keeps a failed destroy retryable and makes completed destroy idempotent", async () => { - seedReady("worker-destroy"); - config.cloudWorkers!.profiles = {}; - let fail = true; - const destroyed: WorkerLifecycleLease[] = []; - const provider = createProvider({ - destroy: async (lease) => { - destroyed.push(lease); - if (fail) { - throw new Error("destroy timeout"); - } - }, - }); - const workerService = createService(provider); - - await expect(workerService.destroy("worker-destroy")).rejects.toMatchObject({ - code: "provider_failure", - } satisfies Partial); - expect(store.get("worker-destroy")).toMatchObject({ - state: "destroying", - lastError: "destroy timeout", - }); - - fail = false; - await workerService.reconcileOnce(); - expect(store.get("worker-destroy")).toMatchObject({ state: "destroyed" }); - await workerService.destroy("worker-destroy"); - expect(destroyed).toEqual([ - { leaseId: "lease:worker-destroy", profile: { region: "test" } }, - { leaseId: "lease:worker-destroy", profile: { region: "test" } }, - ]); - }); - - it("projects live tunnel status and fences the tunnel before provider teardown", async () => { - seedReady("worker-tunnel", undefined, true); - const order: string[] = []; - let tunnelStatus: "stopped" | "connected" = "stopped"; - const tunnelManager = { - status: () => tunnelStatus, - start: vi.fn(async (request) => { - tunnelStatus = "connected"; - return { - environmentId: request.environmentId, - ownerEpoch: request.ownerEpoch, - remoteSocketPath: "/tmp/worker/gateway.sock", - runWorkspaceCommand: vi.fn(), - syncWorkspace: vi.fn(), - stop: async () => {}, - }; - }), - stop: vi.fn(async () => { - tunnelStatus = "stopped"; - order.push("tunnel-stop"); - }), - stopAll: vi.fn(async () => {}), - } as unknown as WorkerTunnelManager; - const provider = createProvider({ - destroy: async () => { - order.push("provider-destroy"); - }, - }); - const workerService = createService(provider, { tunnelManager }); - - await expect( - workerService.startTunnel({ environmentId: "worker-tunnel", ownerEpoch: 0 }), - ).rejects.toThrow("owner credential is not current"); - expect(tunnelManager.start).not.toHaveBeenCalled(); - - await workerService.startTunnel({ - environmentId: "worker-tunnel", - ownerEpoch: 1, - }); - expect(tunnelManager.start).toHaveBeenCalledWith( - expect.objectContaining({ - bundleHash: BUNDLE_HASH, - gateway: { host: "127.0.0.1", port: 18_789 }, - sharedHost: true, - }), - ); - expect(workerService.get("worker-tunnel")).toMatchObject({ tunnelStatus: "connected" }); - - await workerService.destroy("worker-tunnel"); - expect(order).toEqual(["tunnel-stop", "provider-destroy"]); - expect(workerService.get("worker-tunnel")).toMatchObject({ - state: "destroyed", - tunnelStatus: "stopped", - }); - }); - - it("reconciles shared-host isolation for a persisted lease before tunnel startup", async () => { - seedReady("worker-legacy-shared"); - database.db - .prepare("UPDATE worker_environments SET shared_host = NULL WHERE environment_id = ?") - .run("worker-legacy-shared"); - closeOpenClawStateDatabaseForTest(); - database = openOpenClawStateDatabase({ env: { OPENCLAW_STATE_DIR: root } }); - store = createWorkerEnvironmentStore({ database, now: () => nowMs }); - const tunnelManager = { - status: () => "stopped" as const, - start: vi.fn(async (request: Parameters[0]) => ({ - environmentId: request.environmentId, - ownerEpoch: request.ownerEpoch, - remoteSocketPath: "/tmp/worker/gateway.sock", - runWorkspaceCommand: vi.fn(), - syncWorkspace: vi.fn(), - stop: async () => {}, - })), - stop: vi.fn(async () => {}), - stopAll: vi.fn(async () => {}), - } as unknown as WorkerTunnelManager; - let inspectionFails = true; - const provider = createProvider({ - inspect: async () => { - if (inspectionFails) { - throw new Error("provider unavailable"); - } - return { status: "active", sharedHost: true }; - }, - }); - const workerService = createService(provider, { tunnelManager }); - - expect(store.get("worker-legacy-shared")?.sharedHost).toBeNull(); - await workerService.reconcileOnce(); - await expect( - workerService.startTunnel({ environmentId: "worker-legacy-shared", ownerEpoch: 1 }), - ).rejects.toThrow("isolation is not reconciled"); - expect(tunnelManager.start).not.toHaveBeenCalled(); - inspectionFails = false; - await workerService.reconcileOnce(); - expect(store.get("worker-legacy-shared")?.sharedHost).toBe(true); - await workerService.startTunnel({ environmentId: "worker-legacy-shared", ownerEpoch: 1 }); - expect(tunnelManager.start).toHaveBeenCalledWith(expect.objectContaining({ sharedHost: true })); - }); - - it("fences an existing tunnel before changing its shared-host isolation", async () => { - seedReady("worker-isolation-change"); - const stop = vi.fn(async () => { - expect(store.get("worker-isolation-change")?.sharedHost).toBe(false); - }); - const tunnelManager = { - status: () => "connected" as const, - start: vi.fn(), - stop, - stopAll: vi.fn(async () => {}), - } as unknown as WorkerTunnelManager; - const provider = createProvider({ - inspect: async () => ({ status: "active", sharedHost: true }), - }); - - await createService(provider, { tunnelManager }).reconcileOnce(); - - expect(stop).toHaveBeenCalledWith("worker-isolation-change"); - expect(store.get("worker-isolation-change")?.sharedHost).toBe(true); - }); - - it("projects desktop availability only while a desktop lease is observable", () => { - const ready = seedReadyDesktop("worker-desktop-projection"); - const workerService = createService(createProvider()); - expect(workerService.get(ready.environmentId)).toMatchObject({ - desktopAvailable: true, - desktopApps: ["browser", "terminal"], - }); - store.transition({ - environmentId: ready.environmentId, - from: ready.state, - to: "draining", - }); - expect(workerService.get(ready.environmentId)).toMatchObject({ - desktopAvailable: false, - desktopApps: [], - }); - }); - - it("launches only an advertised desktop app through the pinned SSH runtime", async () => { - const record = seedReadyDesktop("worker-desktop-launch"); - const launchApp = vi.fn(async () => {}); - const tunnelManager = { - desktop: { - acquire: vi.fn(), - attachObserver: vi.fn(), - launchApp, - stop: vi.fn(async () => {}), - stopAll: vi.fn(async () => {}), - }, - status: () => "stopped" as const, - start: vi.fn(), - stop: vi.fn(async () => {}), - stopAll: vi.fn(async () => {}), - } as unknown as WorkerTunnelManager; - const workerService = createService(createProvider(), { tunnelManager }); - - await expect( - workerService.launchDesktopApp({ environmentId: record.environmentId, app: "browser" }), - ).resolves.toEqual({ app: "browser", status: "ready" }); - expect(launchApp).toHaveBeenCalledExactlyOnceWith({ - environmentId: record.environmentId, - ownerEpoch: record.ownerEpoch, - ssh: SSH_ENDPOINT, - app: DESKTOP.apps?.[0], - resolveIdentity: expect.any(Function), - }); - }); - - it("rejects missing desktop apps and maps launcher runtime failures to typed errors", async () => { - const record = seedReadyDesktop("worker-desktop-launch-errors"); - const launchApp = vi.fn(async () => { - throw new Error("private SSH launcher detail"); - }); - const tunnelManager = { - desktop: { - acquire: vi.fn(), - attachObserver: vi.fn(), - launchApp, - stop: vi.fn(async () => {}), - stopAll: vi.fn(async () => {}), - }, - status: () => "stopped" as const, - start: vi.fn(), - stop: vi.fn(async () => {}), - stopAll: vi.fn(async () => {}), - } as unknown as WorkerTunnelManager; - const workerService = createService(createProvider(), { tunnelManager }); - - await expect( - workerService.launchDesktopApp({ environmentId: record.environmentId, app: "browser" }), - ).rejects.toMatchObject({ - code: "launcher_failure", - message: "worker desktop browser launcher failed; verify the app is installed and retry", - }); - store.transition({ - environmentId: record.environmentId, - from: record.state, - to: "draining", - }); - await expect( - workerService.launchDesktopApp({ environmentId: record.environmentId, app: "terminal" }), - ).rejects.toMatchObject({ code: "invalid_state" }); - - const browserOnly = seedReadyDesktop("worker-desktop-browser-only", { - ...DESKTOP, - apps: [DESKTOP.apps![0]!], - }); - await expect( - workerService.launchDesktopApp({ - environmentId: browserOnly.environmentId, - app: "terminal", - }), - ).rejects.toMatchObject({ - code: "desktop_app_not_found", - message: "environment does not advertise desktop app: terminal", - }); - }); - - it("acquires a desktop tunnel and mints a one-shot websocket path", async () => { - const record = seedReadyDesktop("worker-desktop-observe"); - const acquire = vi.fn(async () => ({ - localSocketPath: "/tmp/worker-desktop.sock", - vncPassword: "desktop-secret", - })); - const tunnelManager = { - desktop: { - acquire, - attachObserver: vi.fn(), - stop: vi.fn(async () => {}), - stopAll: vi.fn(async () => {}), - }, - status: () => "stopped" as const, - start: vi.fn(), - stop: vi.fn(async () => {}), - stopAll: vi.fn(async () => {}), - } as unknown as WorkerTunnelManager; - const workerService = createService(createProvider(), { tunnelManager }); - - await expect( - workerService.observeDesktop({ environmentId: record.environmentId, control: true }), - ).resolves.toMatchObject({ - transport: "rfb", - wsPath: expect.stringMatching(/^\/worker-desktop\/observe\?token=[a-f0-9]{48}$/u), - expiresAtMs: nowMs + 60_000, - control: true, - vncPassword: "desktop-secret", - }); - expect(acquire).toHaveBeenCalledWith( - expect.objectContaining({ - environmentId: record.environmentId, - ownerEpoch: record.ownerEpoch, - desktop: DESKTOP, - ssh: SSH_ENDPOINT, - resolveIdentity: expect.any(Function), - }), - ); - }); - - it("rejects desktop observe for invalid lifecycle gates and a stopped service", async () => { - const tunnelManager = { - desktop: { - acquire: vi.fn(), - attachObserver: vi.fn(), - stop: vi.fn(async () => {}), - stopAll: vi.fn(async () => {}), - }, - status: () => "stopped" as const, - start: vi.fn(), - stop: vi.fn(async () => {}), - stopAll: vi.fn(async () => {}), - } as unknown as WorkerTunnelManager; - const workerService = createService(createProvider(), { tunnelManager }); - const requested = store.createIntent({ - environmentId: "worker-desktop-requested", - providerId: "fake", - profileId: "development", - profileSnapshot: { settings: { region: "test" } }, - provisionOperationId: "provision:worker-desktop-requested", - }); - seedReady("worker-desktop-missing"); - const destroying = seedReadyDesktop("worker-desktop-destroy-requested"); - store.requestDestroy({ environmentId: destroying.environmentId, state: destroying.state }); - - config.cloudWorkers!.desktop = false; - await expect( - workerService.observeDesktop({ environmentId: requested.environmentId, control: false }), - ).rejects.toMatchObject({ - code: "invalid_state", - message: - "worker desktop observe is disabled; enable the Desktop lab in Control UI Settings -> Labs (config: cloudWorkers.desktop)", - }); - config.cloudWorkers!.desktop = true; - - for (const environmentId of [ - requested.environmentId, - "worker-desktop-missing", - destroying.environmentId, - ]) { - await expect( - workerService.observeDesktop({ environmentId, control: false }), - ).rejects.toMatchObject({ - code: "invalid_state", - message: "environment has no desktop; desktop is a warm-time capability of the profile", - }); - } - await expect( - workerService.observeDesktop({ environmentId: "worker-desktop-unknown", control: false }), - ).rejects.toMatchObject({ code: "environment_not_found" }); - await workerService.stop(); - await expect( - workerService.observeDesktop({ environmentId: destroying.environmentId, control: false }), - ).rejects.toMatchObject({ - code: "invalid_state", - message: "Worker environment service is stopping", - }); - expect(tunnelManager.desktop.acquire).not.toHaveBeenCalled(); - }); - - it("fences a draining tunnel before reporting an unavailable provider", async () => { - seedReady("worker-provider-missing"); - const tunnelManager = { - status: () => "connected" as const, - start: vi.fn(), - stop: vi.fn(async () => {}), - stopAll: vi.fn(async () => {}), - } as unknown as WorkerTunnelManager; - const workerService = createService(createProvider(), { tunnelManager }); - providersEnabled = false; - - await expect(workerService.destroy("worker-provider-missing")).rejects.toMatchObject({ - code: "provider_not_found", - } satisfies Partial); - - expect(tunnelManager.stop).toHaveBeenCalledWith("worker-provider-missing"); - expect(store.get("worker-provider-missing")).toMatchObject({ - state: "draining", - destroyRequestedAtMs: expect.any(Number), - }); - }); - - it("does not hold the environment lock while a tunnel is connecting", async () => { - seedReady("worker-tunnel-pending"); - let rejectStart: ((error: Error) => void) | undefined; - const pendingStart = new Promise((_resolve, reject) => { - rejectStart = reject; - }); - const order: string[] = []; - const tunnelManager = { - status: () => "connecting" as const, - start: vi.fn(() => pendingStart), - stop: vi.fn(async () => { - order.push("tunnel-stop"); - rejectStart?.(new Error("tunnel stopped")); - }), - stopAll: vi.fn(async () => {}), - } as unknown as WorkerTunnelManager; - const provider = createProvider({ - destroy: async () => { - order.push("provider-destroy"); - }, - }); - const workerService = createService(provider, { tunnelManager }); - - const starting = workerService.startTunnel({ - environmentId: "worker-tunnel-pending", - ownerEpoch: 1, - }); - const rejectedStart = expect(starting).rejects.toThrow("tunnel stopped"); - await waitForFast(() => expect(tunnelManager.start).toHaveBeenCalledOnce()); - - await workerService.destroy("worker-tunnel-pending"); - - await rejectedStart; - expect(order).toEqual(["tunnel-stop", "provider-destroy"]); - }); - - it("stops a poisoned tunnel start and returns a typed deadline error", async () => { - vi.useFakeTimers(); - seedReady("worker-tunnel-timeout"); - let signalStarted!: () => void; - const started = new Promise((resolve) => { - signalStarted = resolve; - }); - let rejectStart!: (error: Error) => void; - const pendingStart = new Promise((_resolve, reject) => { - rejectStart = reject; - }); - const tunnelManager = { - status: () => "connecting" as const, - start: vi.fn(() => { - signalStarted(); - return pendingStart; - }), - stop: vi.fn(async () => { - rejectStart(new Error("tunnel stopped")); - }), - stopAll: vi.fn(async () => {}), - } as unknown as WorkerTunnelManager; - const workerService = createService(createProvider(), { tunnelManager }); - - const starting = workerService.startTunnel({ - environmentId: "worker-tunnel-timeout", - ownerEpoch: 1, - }); - const rejected = expect(starting).rejects.toMatchObject({ - code: "provider_failure", - message: expect.stringContaining("did not connect within 3 minutes"), - } satisfies Partial); - await started; - await vi.advanceTimersByTimeAsync(3 * 60_000); - - await rejected; - expect(tunnelManager.stop).toHaveBeenCalledWith("worker-tunnel-timeout", 1); - }); - - it("adopts an unpersisted provision result before destroying", async () => { - const intent = store.createIntent({ - environmentId: "worker-pending-destroy", - providerId: "fake", - profileId: "development", - profileSnapshot: { settings: { region: "test" } }, - provisionOperationId: "provision:pending-destroy", - }); - store.transition({ - environmentId: intent.environmentId, - from: "requested", - to: "provisioning", - }); - const destroyed: WorkerLifecycleLease[] = []; - const provider = createProvider({ - provision: async () => { - expect(store.get(intent.environmentId)?.destroyRequestedAtMs).not.toBeNull(); - return { leaseId: "lease-1", ssh: SSH_ENDPOINT }; - }, - destroy: async (lease) => void destroyed.push(lease), - }); - - const result = await createService(provider).destroy(intent.environmentId); - - expect(result.state).toBe("destroyed"); - expect(destroyed).toEqual([{ leaseId: "lease-1", profile: { region: "test" } }]); - }); - - it("retains teardown intent across an indeterminate provision failure", async () => { - prepareInstallation = vi.fn(async () => { - throw new Error("bundle preparation must not block teardown adoption"); - }); - const intent = store.createIntent({ - environmentId: "worker-pending-destroy-retry", - providerId: "fake", - profileId: "development", - profileSnapshot: { settings: { region: "test" } }, - provisionOperationId: "provision:pending-destroy-retry", - }); - store.transition({ - environmentId: intent.environmentId, - from: "requested", - to: "provisioning", - }); - let provisionFails = true; - const destroyed: WorkerLifecycleLease[] = []; - const provider = createProvider({ - provision: async () => { - if (provisionFails) { - throw new Error("provision outcome unknown"); - } - return { leaseId: "lease-retried", ssh: SSH_ENDPOINT }; - }, - destroy: async (lease) => void destroyed.push(lease), - }); - const workerService = createService(provider); - - providersEnabled = false; - await expect(workerService.destroy(intent.environmentId)).rejects.toMatchObject({ - code: "provider_not_found", - } satisfies Partial); - expect(store.get(intent.environmentId)?.destroyRequestedAtMs).not.toBeNull(); - - providersEnabled = true; - await expect(workerService.destroy(intent.environmentId)).rejects.toMatchObject({ - code: "provider_failure", - } satisfies Partial); - expect(store.get(intent.environmentId)).toMatchObject({ - state: "provisioning", - destroyRequestedAtMs: expect.any(Number), - }); - - provisionFails = false; - await workerService.reconcileOnce(); - expect(store.get(intent.environmentId)?.state).toBe("destroyed"); - expect(destroyed).toEqual([{ leaseId: "lease-retried", profile: { region: "test" } }]); - expect(prepareInstallation).not.toHaveBeenCalled(); - }); - - it("reconciles unrelated leases concurrently", async () => { - seedReady("worker-concurrent-a"); - seedReady("worker-concurrent-b"); - let release: (() => void) | undefined; - const blocked = new Promise((resolve) => { - release = resolve; - }); - const inspected: WorkerLifecycleLease[] = []; - const provider = createProvider({ - inspect: async (lease) => { - inspected.push(lease); - await blocked; - return { status: "active" }; - }, - }); - - const reconciliation = createService(provider).reconcileOnce(); - try { - await waitForFast(() => expect(inspected).toHaveLength(2)); - } finally { - release?.(); - } - await reconciliation; - - expect(new Set(inspected.map(({ leaseId }) => leaseId))).toEqual( - new Set(["lease:worker-concurrent-a", "lease:worker-concurrent-b"]), - ); - }); - - it("waits for timed-out provider work during shutdown", async () => { - let finishProvision: (() => void) | undefined; - const provisionPending = new Promise((resolve) => { - finishProvision = resolve; - }); - const provision = vi.fn(async () => { - await provisionPending; - return { leaseId: "lease-stop-timeout", ssh: SSH_ENDPOINT }; - }); - const stopAll = vi.fn(async () => {}); - const tunnelManager = { - stopAll, - } as unknown as WorkerTunnelManager; - const workerService = createService(createProvider({ provision }), { - providerCallTimeoutMs: 5, - tunnelManager, - }); - const creation = workerService.create("development", "request-stop-provider-timeout"); - const creationResult = expect(creation).rejects.toMatchObject({ - code: "provider_failure", - } satisfies Partial); - let stopped = false; - let stopping: Promise | undefined; - - try { - await waitForFast(() => expect(provision).toHaveBeenCalledOnce()); - await creationResult; - stopping = workerService.stop().then(() => { - stopped = true; - }); - await waitForFast(() => expect(stopAll).toHaveBeenCalledOnce()); - await Promise.resolve(); - expect(stopped).toBe(false); - } finally { - finishProvision?.(); - } - - await stopping; - expect(stopped).toBe(true); - }); - - it("owns and clears one periodic reconciliation timer", async () => { - vi.useFakeTimers(); - const liveEvents = createLiveEvents(); - const workerService = createService(createProvider(), { liveEvents }); - - workerService.start(); - workerService.start(); - expect(liveEvents.start).toHaveBeenCalledOnce(); - expect(vi.getTimerCount()).toBe(1); - await workerService.stop(); - - expect(liveEvents.clear).toHaveBeenCalledTimes(2); - expect(vi.getTimerCount()).toBe(0); - }); - - it("rejects a create queued before service shutdown once its lock is acquired", async () => { - let finishBootstrap: (() => void) | undefined; - const bootstrapPending = new Promise((resolve) => { - finishBootstrap = resolve; - }); - bootstrapWorker = vi.fn(async () => { - await bootstrapPending; - return BOOTSTRAP_RECEIPT; - }); - const provision = vi.fn(createProvider().provision); - const workerService = createService(createProvider({ provision })); - const first = workerService.create("development", "request-queued-before-stop"); - await waitForFast(() => expect(bootstrapWorker).toHaveBeenCalledTimes(1)); - const queued = workerService.create("development", "request-queued-before-stop"); - const queuedResult = expect(queued).rejects.toMatchObject({ - code: "invalid_state", - } satisfies Partial); - - const stopping = workerService.stop(); - finishBootstrap?.(); - - await expect(first).resolves.toMatchObject({ state: "ready" }); - await queuedResult; - await stopping; - expect(provision).toHaveBeenCalledTimes(1); - }); - - it("drains a destroy accepted before service shutdown while it waits for the lock", async () => { - let finishBootstrap: (() => void) | undefined; - const bootstrapPending = new Promise((resolve) => { - finishBootstrap = resolve; - }); - bootstrapWorker = vi.fn(async () => { - await bootstrapPending; - return BOOTSTRAP_RECEIPT; - }); - const destroy = vi.fn(async () => {}); - const workerService = createService(createProvider({ destroy })); - const creation = workerService.create("development", "request-destroy-before-stop"); - await waitForFast(() => expect(bootstrapWorker).toHaveBeenCalledTimes(1)); - const environmentId = store.list()[0]?.environmentId; - expect(environmentId).toBeTruthy(); - const teardown = workerService.destroy(environmentId!); - const teardownResult = expect(teardown).resolves.toMatchObject({ state: "destroyed" }); - - const stopping = workerService.stop(); - finishBootstrap?.(); - - await expect(creation).resolves.toMatchObject({ state: "ready" }); - await teardownResult; - await stopping; - expect(destroy).toHaveBeenCalledTimes(1); - }); - - it("drains accepted operations after reconciliation rejects during shutdown", async () => { - const durableStore = store; - store = { - ...store, - listForReconcile() { - throw new Error("reconcile database read failed"); - }, - }; - let finishBootstrap: (() => void) | undefined; - const bootstrapPending = new Promise((resolve) => { - finishBootstrap = resolve; - }); - bootstrapWorker = vi.fn(async () => { - await bootstrapPending; - return BOOTSTRAP_RECEIPT; - }); - const workerService = createService(createProvider()); - const creation = workerService.create("development", "request-stop-after-reconcile-failure"); - await waitForFast(() => expect(bootstrapWorker).toHaveBeenCalledTimes(1)); - const reconciliation = workerService.reconcileOnce(); - const reconciliationResult = expect(reconciliation).rejects.toThrow( - "reconcile database read failed", - ); - let stopped = false; - const stopping = workerService.stop().then(() => { - stopped = true; - }); - - await reconciliationResult; - await Promise.resolve(); - expect(stopped).toBe(false); - finishBootstrap?.(); - - await expect(creation).resolves.toMatchObject({ state: "ready" }); - await stopping; - expect(stopped).toBe(true); - expect(durableStore.list()).toHaveLength(1); - }); - - it("starts without blocking gateway startup and drains reconciliation on stop", async () => { - seedReady("worker-slow-inspection"); - let finishInspection: (() => void) | undefined; - const inspectionPending = new Promise((resolve) => { - finishInspection = resolve; - }); - const inspect = vi.fn(async () => { - await inspectionPending; - return { status: "active" as const }; - }); - const workerService = createService(createProvider({ inspect })); - - workerService.start(); - await waitForFast(() => expect(inspect).toHaveBeenCalledTimes(1)); - let stopped = false; - const stopping = workerService.stop().then(() => { - stopped = true; - }); - await Promise.resolve(); - expect(stopped).toBe(false); - - finishInspection?.(); - await stopping; - expect(stopped).toBe(true); - await expect(workerService.create("development", "request-after-stop")).rejects.toMatchObject({ - code: "invalid_state", - } satisfies Partial); - await expect(workerService.destroy("worker-slow-inspection")).rejects.toMatchObject({ - code: "invalid_state", - } satisfies Partial); - }); -}); -/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ diff --git a/src/gateway/worker-environments/service.ts b/src/gateway/worker-environments/service.ts index e73d33fd54e1..9bdcf7f0bdd1 100644 --- a/src/gateway/worker-environments/service.ts +++ b/src/gateway/worker-environments/service.ts @@ -1,83 +1,46 @@ -import { createHash } from "node:crypto"; -import { isDeepStrictEqual } from "node:util"; -import { expectDefined } from "@openclaw/normalization-core"; -import { MAX_TIMER_TIMEOUT_MS } from "@openclaw/normalization-core/number-coercion"; -import { - type WorkerAdmissionHandshake, - type WorkerConnectParams, - type WorkerLiveEventParams, - type WorkerProtocolCloseReason, - type WorkerSessionsSendParams, - type WorkerSessionsSpawnParams, - type WorkerSessionToolResult, - type WorkerTranscriptCommitErrorReason, - type WorkerTranscriptCommitParams, - type WorkerTranscriptCommitResult, - WORKER_RPC_SET_VERSION, -} from "../../../packages/gateway-protocol/src/schema/worker-admission.js"; import type { - WorkerInferenceCancelParams, - WorkerInferenceCancelResult, - WorkerInferenceErrorReason, - WorkerInferenceStartParams, - WorkerInferenceStartResult, -} from "../../../packages/gateway-protocol/src/schema/worker-inference.js"; + WorkerAdmissionHandshake, + WorkerSessionsSendParams, + WorkerSessionsSpawnParams, + WorkerSessionToolResult, + WorkerTranscriptCommitErrorReason, + WorkerTranscriptCommitParams, + WorkerTranscriptCommitResult, +} from "../../../packages/gateway-protocol/src/schema/worker-admission.js"; import { onSessionIdentityMutation } from "../../config/sessions/session-accessor.js"; import type { OpenClawConfig } from "../../config/types.js"; import type { SecretRef } from "../../config/types.secrets.js"; -import { validateCloudWorkerProfileSettings } from "../../config/zod-schema.cloud-workers.js"; import { withTimeout } from "../../infra/fs-safe.js"; import { KeyedAsyncQueue } from "../../plugin-sdk/keyed-async-queue.js"; -import { normalizeCapabilityProviderId } from "../../plugins/provider-registry-shared.js"; -import { - WorkerProviderError, - type WorkerLease, - type WorkerProfile, - type WorkerProvider, - type WorkerSshEndpoint, - type WorkerSshIdentity, +import type { + WorkerProfile, + WorkerProvider, + WorkerSshEndpoint, + WorkerSshIdentity, } from "../../plugins/types.js"; -import { safeEqualSecret } from "../../security/secret-equal.js"; import { runTasksWithConcurrency } from "../../utils/run-with-concurrency.js"; -import type { WorkerSessionToolName } from "../../worker/tool-authority.js"; -import { - admitWorkerConnection, - validateWorkerConnectionIdentity, - verifyWorkerAdmissionHandshake, - type ExpectedWorkerBuild, - type WorkerConnectionIdentity, -} from "./admission.js"; +import type { WorkerConnectionIdentity } from "./admission.js"; import type { WorkerInstallationArtifact } from "./bundle.js"; -import { - createWorkerCredentialMaterial, - WORKER_CREDENTIAL_TTL_MS, - type MintedWorkerCredential, - type WorkerCredentialBinding, - type WorkerCredentialDeliveryClaim, -} from "./credential.js"; +import { createWorkerCredentialBroker } from "./credential-broker.js"; +import { createWorkerEnvironmentAccess } from "./environment-access.js"; import { registerWorkerInferenceSessionDrain, type WorkerInferenceSessionDrain, } from "./inference-control-internal.js"; import type { WorkerInferenceStore } from "./inference-store.js"; -import { - createWorkerInferenceManager, - type WorkerInferenceExecutor, - type WorkerInferenceSink, -} from "./inference.js"; -import type { WorkerLiveEventApplicationResult, WorkerLiveEventReceiver } from "./live-events.js"; -import type { WorkerDesktopLaunchResult, WorkerDesktopObserveResult } from "./service-contract.js"; -import { requireWorkerLeaseStatus, requireWorkerLease } from "./service-validation.js"; +import { createWorkerInferenceManager, type WorkerInferenceExecutor } from "./inference.js"; +import type { WorkerLiveEventReceiver } from "./live-events.js"; +import type { WorkerSessionPlacementGate } from "./placement-worker-gate.js"; +import { createWorkerProviderLifecycle } from "./provider-lifecycle.js"; import type { WorkerEnvironmentState } from "./state.js"; -import { - type WorkerEnvironmentRecord, - type WorkerEnvironmentStore, - type WorkerEnvironmentTransitionPatch as TransitionPatch, - WorkerSessionAlreadyAttachedError, +import type { + WorkerEnvironmentRecord, + WorkerEnvironmentStore, + WorkerEnvironmentTransitionPatch as TransitionPatch, } from "./store.js"; -import type { WorkerTunnelRequest } from "./tunnel-contract.js"; -import type { WorkerTunnelHandle, WorkerTunnelManager } from "./tunnel.js"; +import type { WorkerTunnelManager } from "./tunnel.js"; import { boundedWorkerError as boundedError } from "./worker-error.js"; +import { createWorkerTurnRpc } from "./worker-turn-rpc.js"; type WorkerEnvironmentServiceErrorCode = | "profile_not_found" @@ -102,30 +65,6 @@ class WorkerEnvironmentServiceError extends Error { const serviceError = (code: WorkerEnvironmentServiceErrorCode, message: string) => new WorkerEnvironmentServiceError(code, message); -const ORPHANED_LEASE_ERROR = "Worker provider no longer recognizes the lease"; -// One poisoned SSH attempt must not hold every later dispatch on the same owner epoch forever. -const TUNNEL_START_TIMEOUT_MS = 3 * 60_000; - -function requireProviderProvisionTimeoutMs(timeoutMs: number | undefined): number | undefined { - if (timeoutMs === undefined) { - return undefined; - } - if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > MAX_TIMER_TIMEOUT_MS) { - throw new WorkerProviderError( - `Worker provider provision timeout must be an integer from 1 through ${MAX_TIMER_TIMEOUT_MS}ms`, - ); - } - return timeoutMs; -} - -function workerEnvironmentIdempotencyDigest(idempotencyKey: string): string { - return createHash("sha256").update(idempotencyKey).digest("hex"); -} - -export function workerEnvironmentIdForIdempotencyKey(idempotencyKey: string): string { - const digest = workerEnvironmentIdempotencyDigest(idempotencyKey); - return `worker:${digest.slice(0, 32)}`; -} type WorkerEnvironmentServiceOptions = { store: WorkerEnvironmentStore; @@ -159,7 +98,10 @@ type WorkerEnvironmentServiceOptions = { applyTranscriptCommit?: (params: { identity: WorkerConnectionIdentity; request: WorkerTranscriptCommitParams; - }) => Promise; + }) => Promise< + | { ok: true; result: WorkerTranscriptCommitResult } + | { ok: false; reason: WorkerTranscriptCommitErrorReason } + >; liveEvents?: Pick< WorkerLiveEventReceiver, "apply" | "bindSession" | "clear" | "clearEnvironment" | "rotateCredential" | "start" @@ -184,99 +126,12 @@ type WorkerEnvironmentServiceOptions = { ) => Promise; }; -export type WorkerPlacementBinding = Readonly<{ - sessionId: string; - environmentId: string; - ownerEpoch: number; -}>; - -export type WorkerPlacementTurnBinding = WorkerPlacementBinding & - Readonly<{ - runId: string; - }>; - -type WorkerProcessTurnBinding = WorkerPlacementTurnBinding & { - credentialHash: string; -}; - -type WorkerTerminalTurnFence = WorkerProcessTurnBinding & { - transcriptSeq: number; - liveSeq: number; -}; - -type WorkerPendingTerminalTurnFence = WorkerProcessTurnBinding & { - terminalLiveSeq: number; -}; - -type WorkerTurnRequest = - | { kind: "inference" } - | { kind: "live"; seq: number } - | { kind: "transcript"; seq: number } - | { kind: "session-tool" }; - -export type WorkerSessionPlacementGate = { - hasWorkerTurn(binding: WorkerPlacementBinding): boolean; - validateWorkerTurn(binding: WorkerPlacementTurnBinding): boolean; - isWorkerTurnToolAuthorized(binding: WorkerPlacementTurnBinding, toolName: string): boolean; - updateAckCursors( - binding: WorkerPlacementTurnBinding & { - transcriptSeq?: number; - liveSeq?: number; - workspaceResultPending?: boolean; - }, - ): void; -}; - -type WorkerTranscriptCommitApplicationResult = - | { ok: true; result: WorkerTranscriptCommitResult } - | { ok: false; reason: WorkerTranscriptCommitErrorReason }; - -type WorkerTranscriptCommitServiceResult = - | WorkerTranscriptCommitApplicationResult - | { ok: false; closeReason: WorkerProtocolCloseReason }; - -type WorkerLiveEventServiceResult = - | WorkerLiveEventApplicationResult - | { ok: false; closeReason: WorkerProtocolCloseReason }; - -type WorkerInferenceStartServiceResult = - | { - ok: true; - result: WorkerInferenceStartResult; - launch: () => void; - } - | { ok: false; reason: WorkerInferenceErrorReason } - | { ok: false; closeReason: WorkerProtocolCloseReason }; - -type WorkerInferenceCancelServiceResult = - | { ok: true; result: WorkerInferenceCancelResult } - | { ok: false; reason: WorkerInferenceErrorReason } - | { ok: false; closeReason: WorkerProtocolCloseReason }; - -type WorkerSessionToolServiceResult = - | { ok: true; result: WorkerSessionToolResult } - | { ok: false; closeReason: WorkerProtocolCloseReason } - | { ok: false; reason: WorkerProtocolCloseReason }; - -function requireWorkerProfile(value: unknown): WorkerProfile { - const error = validateCloudWorkerProfileSettings(value); - if (error) { - throw serviceError("invalid_profile", error); - } - return value as WorkerProfile; -} - export function createWorkerEnvironmentService(options: WorkerEnvironmentServiceOptions) { const { store } = options; - const tunnels = options.tunnelManager; const warn = (message: string) => options.logger?.warn(message); const operations = new KeyedAsyncQueue(); const providerOperations = new KeyedAsyncQueue(); const activeOperations = new Set>(); - const pendingCredentials = new Map(); - const observedAckCursors = new Map(); - const pendingTerminalTurnFences = new Map(); - const terminalTurnFences = new Map(); const now = options.now ?? Date.now; const inference = createWorkerInferenceManager({ execute: options.executeInference, @@ -292,140 +147,16 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService let unsubscribeSessionIdentityMutation: (() => void) | undefined; let stopping = false; - const placementBinding = ( - identity: WorkerConnectionIdentity, - ): WorkerPlacementTurnBinding | undefined => { - if (!identity.sessionId || !identity.runId) { - return undefined; - } - return { - sessionId: identity.sessionId, - environmentId: identity.environmentId, - ownerEpoch: identity.ownerEpoch, - runId: identity.runId, - }; - }; + const inState = (record: WorkerEnvironmentRecord, ...states: WorkerEnvironmentState[]) => + states.includes(record.state); - const processTurnBinding = ( - identity: WorkerConnectionIdentity, - ): WorkerProcessTurnBinding | undefined => { - const placement = placementBinding(identity); - return placement ? { ...placement, credentialHash: identity.credentialHash } : undefined; - }; - - const matchesTurnBinding = ( - left: WorkerProcessTurnBinding, - right: WorkerProcessTurnBinding, - ): boolean => - left.sessionId === right.sessionId && - left.environmentId === right.environmentId && - left.ownerEpoch === right.ownerEpoch && - left.runId === right.runId && - safeEqualSecret(left.credentialHash, right.credentialHash); - - const recordAckCursor = ( - binding: WorkerProcessTurnBinding, - cursor: { transcriptSeq: number } | { liveSeq: number }, - ): WorkerTerminalTurnFence => { - const current = observedAckCursors.get(binding.sessionId); - const currentTurn = current && matchesTurnBinding(current, binding) ? current : undefined; - const next: WorkerTerminalTurnFence = { - ...binding, - transcriptSeq: - "transcriptSeq" in cursor - ? Math.max(currentTurn?.transcriptSeq ?? 0, cursor.transcriptSeq) - : (currentTurn?.transcriptSeq ?? 0), - liveSeq: - "liveSeq" in cursor - ? Math.max(currentTurn?.liveSeq ?? 0, cursor.liveSeq) - : (currentTurn?.liveSeq ?? 0), - }; - observedAckCursors.set(binding.sessionId, next); - return next; - }; - - const observedAckCursorFor = ( - binding: WorkerProcessTurnBinding, - ): WorkerTerminalTurnFence | undefined => { - const observed = observedAckCursors.get(binding.sessionId); - return observed && matchesTurnBinding(observed, binding) ? observed : undefined; - }; - - const validateWorkerPlacement = ( - identity: WorkerConnectionIdentity, - ): { durableClaim: boolean; valid: boolean } => { - if (!options.placementStore) { - return { durableClaim: false, valid: true }; - } - if (identity.sessionId === null && identity.runId === null) { - return { durableClaim: false, valid: true }; - } - const binding = placementBinding(identity); - const valid = binding ? options.placementStore.validateWorkerTurn(binding) : false; - return { durableClaim: valid, valid }; - }; - - const isTerminalLiveEvent = (request: WorkerLiveEventParams): boolean => - request.event.kind === "lifecycle" && - (request.event.payload.phase === "finishing" || - request.event.payload.phase === "end" || - (request.event.payload.phase === "error" && - (request.event.payload.aborted === true || - request.event.payload.fallbackExhaustedFailure === true))); - - const project = (record: WorkerEnvironmentRecord) => { - const desktopAvailable = - inState(record, "ready", "idle", "attached") && record.desktop !== null; - return { - ...record, - ...((record.state === "failed" || record.state === "orphaned") && record.lastError - ? { error: boundedError(record.lastError) } - : {}), - desktopAvailable, - desktopApps: desktopAvailable - ? (record.desktop?.apps?.map((app) => app.id).toSorted() ?? []) - : [], - tunnelStatus: tunnels?.status(record.environmentId) ?? ("stopped" as const), - }; - }; - - const move = ( - r: WorkerEnvironmentRecord, - to: WorkerEnvironmentState, - patch?: TransitionPatch, - ) => { - const next = store.transition({ environmentId: r.environmentId, from: r.state, to, patch }); - if (to !== "ready" && to !== "idle" && to !== "attached") { - pendingCredentials.delete(r.environmentId); - } - if (to !== "attached") { - inference.cancelEnvironment(r.environmentId); - options.liveEvents?.clearEnvironment(r.environmentId); - } - return next; - }; - - const saveError = (r: WorkerEnvironmentRecord, error: unknown) => { - // Once bootstrap failure owns the terminal outcome, preserve that causal error across - // transient provider/inspection failures so the final failed row stays actionable. - if (r.teardownTerminalState === "failed" && r.lastError) { - return r; - } - return store.recordError({ - environmentId: r.environmentId, - state: r.state, - error: boundedError(error), - }); - }; - - const inState = (r: WorkerEnvironmentRecord, ...states: WorkerEnvironmentState[]) => - states.includes(r.state); const trackOperation = (operation: Promise) => { activeOperations.add(operation); const release = () => activeOperations.delete(operation); void operation.then(release, release); return operation; }; + const withLock = (environmentId: string, task: () => Promise) => trackOperation(operations.enqueue(environmentId, task)); @@ -471,999 +202,105 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService } }; - // Durable profile settings keep lifecycle routing stable across config edits and restarts. - const lifecycleLease = (record: WorkerEnvironmentRecord, leaseId: string) => ({ - leaseId, - profile: requireWorkerProfile(record.profileSnapshot.settings), + const move = ( + record: WorkerEnvironmentRecord, + to: WorkerEnvironmentState, + patch?: TransitionPatch, + ) => { + const next = store.transition({ + environmentId: record.environmentId, + from: record.state, + to, + patch, + }); + if (to !== "ready" && to !== "idle" && to !== "attached") { + credentialBroker.clearEnvironment(record.environmentId); + } + if (to !== "attached") { + inference.cancelEnvironment(record.environmentId); + options.liveEvents?.clearEnvironment(record.environmentId); + } + return next; + }; + + const saveError = (record: WorkerEnvironmentRecord, error: unknown) => { + // Once bootstrap failure owns the terminal outcome, preserve that causal error across + // transient provider/inspection failures so the final failed row stays actionable. + if (record.teardownTerminalState === "failed" && record.lastError) { + return record; + } + return store.recordError({ + environmentId: record.environmentId, + state: record.state, + error: boundedError(error), + }); + }; + + const credentialBroker = createWorkerCredentialBroker({ + store, + prepareInstallation: options.prepareInstallation, + tunnelManager: options.tunnelManager, + workerCredentialTtlMs: options.workerCredentialTtlMs, + generateWorkerCredential: options.generateWorkerCredential, + liveEvents: options.liveEvents, + placementStore: options.placementStore, + now, + isStopping: () => stopping, + cancelInferenceEnvironment: (environmentId) => inference.cancelEnvironment(environmentId), + inState, + move, + serviceError, + withLock, }); - const identityResolverFor = ( - record: WorkerEnvironmentRecord, - provider: WorkerProvider, - leaseId: string, - ) => { - const profile = requireWorkerProfile(record.profileSnapshot.settings); - const resolveSshIdentity = options.resolveSshIdentity; - return async (keyRef: SecretRef) => { - if (!resolveSshIdentity) { - throw new Error("Worker SSH identity resolution is unavailable"); - } - return await callProvider(record.environmentId, () => - resolveSshIdentity({ provider, leaseId, profile, keyRef }), - ); - }; - }; + const providerLifecycle = createWorkerProviderLifecycle({ + store, + getConfig: options.getConfig, + resolveProvider: options.resolveProvider, + prepareInstallation: options.prepareInstallation, + bootstrapWorker: options.bootstrapWorker, + resolveSshIdentity: options.resolveSshIdentity, + providerCallTimeoutMs: options.providerCallTimeoutMs, + tunnelManager: options.tunnelManager, + credentialBroker, + callBootstrap, + callProvider, + inState, + isServiceError: (error, code) => + error instanceof WorkerEnvironmentServiceError && error.code === code, + isStopping: () => stopping, + move, + saveError, + serviceError, + withLock, + }); - const providerFor = (providerId: string): WorkerProvider => { - const provider = options.resolveProvider(providerId); - if (provider) { - return provider; - } - throw serviceError("provider_not_found", `Worker provider is unavailable: ${providerId}`); - }; + const environmentAccess = createWorkerEnvironmentAccess({ + store, + getConfig: options.getConfig, + tunnelManager: options.tunnelManager, + resolveWorkerGateway: options.resolveWorkerGateway, + now, + identityResolverFor: providerLifecycle.identityResolverFor, + inState, + isStopping: () => stopping, + providerFor: providerLifecycle.providerFor, + serviceError, + withLock, + }); - const installFor = (record: WorkerEnvironmentRecord): WorkerInstallationArtifact["install"] => { - const install = record.profileSnapshot.install; - if (install === undefined || install === "bundle") { - return "bundle"; - } - if (install === "npm") { - return "npm"; - } - throw serviceError("invalid_profile", "Worker profile has an invalid install method"); - }; - - const prepareInstallation = (record: WorkerEnvironmentRecord) => - options.prepareInstallation(installFor(record)); - - const credentialExpiry = () => { - const ttlMs = options.workerCredentialTtlMs ?? WORKER_CREDENTIAL_TTL_MS; - if (!Number.isSafeInteger(ttlMs) || ttlMs < 1) { - throw serviceError("invalid_state", "Worker credential lifetime is invalid"); - } - const expiresAtMs = now() + ttlMs; - if (!Number.isSafeInteger(expiresAtMs)) { - throw serviceError("invalid_state", "Worker credential expiry is out of range"); - } - return expiresAtMs; - }; - - const credentialMaterial = () => createWorkerCredentialMaterial(options.generateWorkerCredential); - - const grantFrom = (params: { - credential: string; - record: ReturnType; - }): MintedWorkerCredential => { - const record = params.record; - if (!record) { - throw serviceError("invalid_state", "Worker credential persistence failed"); - } - return { - credential: params.credential, - deliveryId: record.credentialHash, - environmentId: record.environmentId, - bundleHash: record.bundleHash, - sessionId: record.sessionId, - rpcSetVersion: record.rpcSetVersion, - ownerEpoch: record.ownerEpoch, - expiresAtMs: record.expiresAtMs, - }; - }; - - const mintCredentialLocked = ( - request: WorkerCredentialBinding, - ): { credentialHash: string; grant: MintedWorkerCredential } => { - const previous = store.getCredential(request.environmentId); - if (previous) { - inference.cancelEnvironment(request.environmentId); - } - const material = credentialMaterial(); - const credential = { - environmentId: request.environmentId, - expectedOwnerEpoch: request.ownerEpoch, - credentialHash: material.credentialHash, - sessionId: request.sessionId, - rpcSetVersion: WORKER_RPC_SET_VERSION, - expiresAtMs: credentialExpiry(), - }; - const record = store.renewCredential(credential); - return { - credentialHash: material.credentialHash, - grant: grantFrom({ credential: material.credential, record }), - }; - }; - - const stageCredential = (grant: MintedWorkerCredential): MintedWorkerCredential => { - pendingCredentials.set(grant.environmentId, grant); - return grant; - }; - - const finishProvenDestroy = (record: WorkerEnvironmentRecord) => { - const destroying = beginDestroy(record); - if (destroying.teardownTerminalState !== "failed") { - return move(destroying, "destroyed"); - } - return move(destroying, "failed", { - leaseId: null, - sshEndpoint: null, - sharedHost: false, - lastError: destroying.lastError ?? "Worker bootstrap failed after provider teardown", - }); - }; - - const failBootstrap = async ( - record: WorkerEnvironmentRecord, - leaseId: string, - provider: WorkerProvider, - error: unknown, - ): Promise => { - const detail = boundedError(error); - const requested = store.requestDestroy({ - environmentId: record.environmentId, - state: record.state, - terminalState: "failed", - lastError: detail, - }); - const draining = move(requested, "draining", { lastError: detail }); - await tunnels?.stop(record.environmentId); - const destroying = move(draining, "destroying", { lastError: detail }); - try { - await callProvider(record.environmentId, () => - provider.destroy(lifecycleLease(record, leaseId)), - ); - } catch (cleanupError) { - // An indeterminate destroy must remain retryable; never hide a possibly-live paid lease - // behind terminal failed state. - saveError( - destroying, - new Error(`${detail}; provider teardown pending: ${boundedError(cleanupError)}`), - ); - throw serviceError( - "bootstrap_failure", - `Worker bootstrap failed; teardown is pending: ${detail}`, - ); - } - finishProvenDestroy(destroying); - throw serviceError("bootstrap_failure", `Worker bootstrap failed: ${detail}`); - }; - - const finishBootstrap = async ( - record: WorkerEnvironmentRecord, - provider: WorkerProvider, - installation: WorkerInstallationArtifact, - ) => { - if (record.state !== "bootstrapping" || !record.leaseId || !record.sshEndpoint) { - throw serviceError("invalid_state", "Worker bootstrap requires a provisioned SSH lease"); - } - let receipt: WorkerAdmissionHandshake; - try { - receipt = await callBootstrap((signal) => - options.bootstrapWorker({ - operationId: record.provisionOperationId, - sshEndpoint: record.sshEndpoint, - installation, - resolveIdentity: identityResolverFor(record, provider, record.leaseId), - signal, - }), - ); - if (!verifyWorkerAdmissionHandshake(receipt, installation)) { - throw new Error("Worker bootstrap receipt does not match the expected build identity"); - } - } catch (error) { - return await failBootstrap(record, record.leaseId, provider, error); - } - const material = credentialMaterial(); - // Receipt, owner epoch, and credential hash commit together. A failed write leaves the - // durable lease bootstrapping so reconcile can retry without admitting a partial identity. - const ready = move(record, "ready", { - bootstrapReceipt: receipt, - credential: { - credentialHash: material.credentialHash, - sessionId: null, - rpcSetVersion: WORKER_RPC_SET_VERSION, - expiresAtMs: credentialExpiry(), - }, - }); - const grant = grantFrom({ - credential: material.credential, - record: store.getCredential(record.environmentId), - }); - stageCredential(grant); - return ready; - }; - - const finishProvision = async ( - record: WorkerEnvironmentRecord, - provider: WorkerProvider, - preparedInstallation?: WorkerInstallationArtifact, - ) => { - let lease: WorkerLease; - try { - const profile = requireWorkerProfile(record.profileSnapshot.settings); - const providerTimeoutMs = - options.providerCallTimeoutMs === undefined - ? requireProviderProvisionTimeoutMs(provider.resolveProvisionTimeoutMs?.(profile)) - : undefined; - lease = requireWorkerLease( - await callProvider( - record.environmentId, - () => provider.provision(profile, record.provisionOperationId), - providerTimeoutMs, - ), - ); - } catch (error) { - const detail = boundedError(error); - if ( - error instanceof WorkerProviderError || - (error instanceof WorkerEnvironmentServiceError && error.code === "invalid_profile") - ) { - move(record, "failed", { lastError: detail }); - throw serviceError("invalid_profile", `Worker provider rejected profile: ${detail}`); - } - saveError(record, error); - throw serviceError("provider_failure", `Worker provider operation failed: ${detail}`); - } - // A timeout can happen after allocation; retain the same operation id for safe replay. - const patch = { - leaseId: lease.leaseId, - sshEndpoint: lease.ssh, - sharedHost: lease.sharedHost === true, - desktop: lease.desktop ?? null, - }; - const bootstrapping = move(record, "bootstrapping", patch); - if (record.destroyRequestedAtMs !== null) { - return bootstrapping; - } - let installation = preparedInstallation; - if (!installation) { - try { - // A persisted provisioning row can represent an allocation whose response was lost. - // Replay the idempotent provider operation before packaging can terminalize that lease. - installation = await prepareInstallation(bootstrapping); - } catch (error) { - return await failBootstrap(bootstrapping, lease.leaseId, provider, error); - } - } - return finishBootstrap(bootstrapping, provider, installation); - }; - - const resumeProvision = async ( - record: WorkerEnvironmentRecord, - provider = providerFor(record.providerId), - ) => { - let installation: WorkerInstallationArtifact | undefined; - if (record.state === "requested" && record.destroyRequestedAtMs === null) { - try { - // Fresh requests package before allocation. Once provisioning is durable, provider replay - // must happen first because the previous response may have been lost after allocation. - installation = await prepareInstallation(record); - } catch (error) { - const detail = boundedError(error); - move(record, "failed", { lastError: detail }); - throw serviceError( - "bootstrap_failure", - `Worker installation preparation failed: ${detail}`, - ); - } - } - const provisioning = record.state === "requested" ? move(record, "provisioning") : record; - return finishProvision(provisioning, provider, installation); - }; - - const cancelRequested = (record: WorkerEnvironmentRecord) => - move(record, "failed", { lastError: "Provisioning canceled before provider allocation" }); - - const beginDrain = (record: WorkerEnvironmentRecord) => { - const failurePatch = - record.teardownTerminalState === "failed" ? { lastError: record.lastError } : undefined; - return inState(record, "bootstrapping", "ready", "attached", "idle") - ? move(record, "draining", failurePatch) - : record; - }; - - const beginDestroy = (record: WorkerEnvironmentRecord) => { - const failurePatch = - record.teardownTerminalState === "failed" ? { lastError: record.lastError } : undefined; - const draining = beginDrain(record); - if (draining.state === "draining") { - return move(draining, "destroying", failurePatch); - } - if (draining.state === "destroying") { - return draining; - } - throw serviceError("invalid_state", `Cannot destroy worker in state: ${record.state}`); - }; - - const finishDestroy = async (r: WorkerEnvironmentRecord, provider?: WorkerProvider) => { - if (!r.leaseId) { - throw serviceError("invalid_state", "Worker environment has no lease"); - } - const leaseId = r.leaseId; - const draining = beginDrain(r); - await tunnels?.stop(r.environmentId); - const owningProvider = provider ?? providerFor(r.providerId); - const destroying = beginDestroy(draining); - try { - await callProvider(r.environmentId, () => owningProvider.destroy(lifecycleLease(r, leaseId))); - } catch (error) { - saveError(destroying, error); - throw serviceError("provider_failure", "Worker provider operation failed"); - } - return finishProvenDestroy(destroying); - }; - - const ensurePendingCredential = (record: WorkerEnvironmentRecord, sessionId: string | null) => { - const credential = store.getCredential(record.environmentId); - const pending = pendingCredentials.get(record.environmentId); - const credentialHasDurableTurn = - credential?.deliveredAtMs !== null && - credential?.ownerEpoch === record.ownerEpoch && - credential.sessionId === sessionId && - sessionId !== null && - options.placementStore?.hasWorkerTurn({ - sessionId, - environmentId: record.environmentId, - ownerEpoch: record.ownerEpoch, - }) === true; - const credentialIsCurrent = - credential?.ownerEpoch === record.ownerEpoch && - credential.sessionId === sessionId && - (credential.expiresAtMs > now() || credentialHasDurableTurn); - const pendingIsCurrent = - credentialIsCurrent && - pending?.deliveryId === credential.credentialHash && - pending.ownerEpoch === record.ownerEpoch && - pending.sessionId === sessionId; - if (credentialIsCurrent && credential.deliveredAtMs !== null) { - pendingCredentials.delete(record.environmentId); - return; - } - if (pendingIsCurrent) { - return; - } - pendingCredentials.delete(record.environmentId); - const minted = mintCredentialLocked({ - environmentId: record.environmentId, - ownerEpoch: record.ownerEpoch, - sessionId, - }); - stageCredential(minted.grant); - if (sessionId && credential?.ownerEpoch === record.ownerEpoch) { - options.liveEvents?.rotateCredential({ - credentialHash: minted.credentialHash, - environmentId: record.environmentId, - previousCredentialHash: credential.credentialHash, - runEpoch: record.ownerEpoch, - sessionId, - }); - } - }; - - const reconcileRecord = async (initialRecord: WorkerEnvironmentRecord): Promise => { - let record = initialRecord; - if (record.state === "requested" && record.destroyRequestedAtMs !== null) { - return void cancelRequested(record); - } - let currentBundle: WorkerInstallationArtifact | undefined; - if (record.destroyRequestedAtMs === null && inState(record, "ready", "idle", "attached")) { - try { - currentBundle = await options.prepareInstallation("bundle"); - if ( - record.bootstrapReceipt && - verifyWorkerAdmissionHandshake(record.bootstrapReceipt, currentBundle) - ) { - const sessionId = record.state === "attached" ? record.attachedSessionIds[0] : null; - if (record.state !== "attached" || sessionId) { - ensurePendingCredential(record, sessionId ?? null); - record = store.get(record.environmentId) ?? record; - } - } - } catch { - // Provider inspection and the state-specific path below retain their existing retry policy. - } - } - let provider: WorkerProvider; - try { - provider = providerFor(record.providerId); - } catch (error) { - saveError(record, error); - return; - } - const leaseId = record.leaseId; - if (!leaseId) { - const provisioned = await resumeProvision(record, provider).catch(() => undefined); - if (provisioned?.state === "bootstrapping") { - await finishDestroy(provisioned, provider).catch(() => undefined); - } - return; - } - const inspection = await callProvider(record.environmentId, () => - provider.inspect(lifecycleLease(record, leaseId)), - ) - .then(requireWorkerLeaseStatus) - .catch((error: unknown) => { - saveError(record, error); - return undefined; - }); - if (!inspection) { - return; - } - const { status } = inspection; - const teardownExpected = record.destroyRequestedAtMs !== null || record.state === "destroying"; - if (status === "destroyed" || (status === "unknown" && teardownExpected)) { - const requested = - record.destroyRequestedAtMs === null - ? store.requestDestroy({ - environmentId: record.environmentId, - state: record.state, - ...(status === "destroyed" && !teardownExpected - ? { - terminalState: "failed", - lastError: "Worker environment disappeared before teardown was requested", - } - : {}), - }) - : record; - const draining = beginDrain(requested); - await tunnels?.stop(record.environmentId); - finishProvenDestroy(draining); - return; - } - if (status === "unknown") { - const draining = - record.state === "draining" - ? record - : move(record, "draining", { lastError: ORPHANED_LEASE_ERROR }); - await tunnels?.stop(record.environmentId); - move(draining, "orphaned", { lastError: ORPHANED_LEASE_ERROR }); - return; - } - const inspectedSharedHost = inspection.sharedHost === true; - if (record.sharedHost !== null && record.sharedHost !== inspectedSharedHost) { - // Workspace actions capture isolation at tunnel creation. Fence the old actions before - // committing a provider-owned change so no reconciliation can use stale host scope. - await tunnels?.stop(record.environmentId); - } - record = store.reconcileSharedHost({ - environmentId: record.environmentId, - state: record.state, - leaseId, - sharedHost: inspectedSharedHost, - }); - if (record.destroyRequestedAtMs !== null) { - await finishDestroy(record, provider).catch(() => undefined); - return; - } - if (record.state === "attached") { - if ( - currentBundle && - (!record.bootstrapReceipt || - !verifyWorkerAdmissionHandshake(record.bootstrapReceipt, currentBundle)) - ) { - // A new Gateway build rejects the old worker at admission. This is expected lifecycle - // teardown, not a bootstrap failure. `leaseId` above came from this record, so provider - // inspection and destruction share the same durable lease identity. - await finishDestroy(record, provider).catch(() => undefined); - } - return; - } - if (record.state === "draining" && record.destroyRequestedAtMs === null) { - // Draining without destroy intent is durable provider-loss cleanup. - await tunnels?.stop(record.environmentId); - move(record, "orphaned", { lastError: record.lastError ?? ORPHANED_LEASE_ERROR }); - return; - } - if (inState(record, "bootstrapping", "ready", "idle")) { - let installation = currentBundle; - try { - // Bundle identity is local and canonical for both install channels. A matching admitted - // receipt must not depend on npm registry availability during routine reconciliation. - installation ??= await options.prepareInstallation("bundle"); - } catch (error) { - if (record.bootstrapReceipt && inState(record, "ready", "idle")) { - saveError(record, error); - return; - } - await failBootstrap(record, leaseId, provider, error).catch(() => undefined); - return; - } - if ( - record.bootstrapReceipt && - verifyWorkerAdmissionHandshake(record.bootstrapReceipt, installation) - ) { - ensurePendingCredential(record, null); - return; - } - if (installFor(record) === "npm") { - try { - installation = await options.prepareInstallation("npm"); - } catch (error) { - await failBootstrap(record, leaseId, provider, error).catch(() => undefined); - return; - } - } - const bootstrapping = - record.state === "bootstrapping" ? record : move(record, "bootstrapping"); - await tunnels?.stop(record.environmentId, record.ownerEpoch); - await finishBootstrap(bootstrapping, provider, installation).catch(() => undefined); - return; - } - if (inState(record, "draining", "destroying")) { - await finishDestroy(record, provider).catch(() => undefined); - } - }; - - const create = async (profileId: string, idempotencyKey: string) => { - return await createWithProfile(profileId, idempotencyKey); - }; - - const createWithProfile = async ( - profileId: string, - idempotencyKey: string, - inherited?: { - providerId: string; - profileSnapshot: WorkerProfile; - }, - ) => { - if (stopping) { - throw serviceError("invalid_state", "Worker environment service is stopping"); - } - const normalizedProfileId = profileId.trim(); - if (!normalizedProfileId || normalizedProfileId !== profileId) { - throw serviceError("invalid_profile", "Worker profile id must be non-empty and trimmed"); - } - const digest = workerEnvironmentIdempotencyDigest(idempotencyKey); - const environmentId = `worker:${digest.slice(0, 32)}`; - return withLock(environmentId, async () => { - if (stopping) { - throw serviceError("invalid_state", "Worker environment service is stopping"); - } - const existing = store.get(environmentId); - if (existing) { - if ( - existing.profileId !== normalizedProfileId || - (inherited !== undefined && - (existing.providerId !== inherited.providerId || - !isDeepStrictEqual(existing.profileSnapshot, inherited.profileSnapshot))) - ) { - throw serviceError("invalid_profile", "Idempotency key belongs to another profile"); - } - if (existing.destroyRequestedAtMs !== null) { - return existing; - } - if (!existing.leaseId && inState(existing, "requested", "provisioning")) { - return resumeProvision(existing); - } - return existing; - } - let provider: WorkerProvider; - let providerId: string; - let profileSnapshot: WorkerProfile; - if (inherited) { - providerId = normalizeCapabilityProviderId(inherited.providerId) ?? inherited.providerId; - if (providerId !== inherited.providerId) { - throw serviceError("invalid_profile", "Inherited worker provider id is not canonical"); - } - provider = providerFor(providerId); - const resolvedProviderId = normalizeCapabilityProviderId(provider.id) ?? provider.id; - if (resolvedProviderId !== providerId) { - throw serviceError("invalid_profile", "Inherited worker provider identity changed"); - } - profileSnapshot = requireWorkerProfile(inherited.profileSnapshot); - } else { - const profiles = options.getConfig().cloudWorkers?.profiles; - if (!profiles || !Object.hasOwn(profiles, normalizedProfileId)) { - throw serviceError("profile_not_found", `Unknown worker profile: ${normalizedProfileId}`); - } - const profile = expectDefined( - profiles[normalizedProfileId], - "profiles entry at normalized profile id", - ); - provider = providerFor(profile.provider); - providerId = normalizeCapabilityProviderId(provider.id) ?? provider.id; - const settings = requireWorkerProfile(profile.settings ?? {}); - profileSnapshot = requireWorkerProfile({ - install: profile.install ?? "bundle", - settings, - }); - } - const intent = store.createIntent({ - environmentId, - providerId, - profileId: normalizedProfileId, - profileSnapshot, - provisionOperationId: `provision:v2:${digest}`, - }); - return resumeProvision(intent, provider); - }); - }; - - const destroy = async ( - environmentId: string, - destroyOptions: { requireUnattached?: boolean } = {}, - ) => { - if (stopping) { - throw serviceError("invalid_state", "Worker environment service is stopping"); - } - return withLock(environmentId, async () => { - let record = store.get(environmentId); - if (!record) { - throw serviceError("environment_not_found", `Unknown worker environment: ${environmentId}`); - } - if (inState(record, "destroyed", "failed", "orphaned")) { - return record; - } - if (destroyOptions.requireUnattached && record.attachedSessionIds.length > 0) { - throw serviceError( - "invalid_state", - "Attached cloud workers must be stopped through sessions.reclaim", - ); - } - record = store.requestDestroy({ environmentId, state: record.state }); - if (record.state === "requested") { - return cancelRequested(record); - } - if (record.leaseId) { - record = beginDrain(record); - } - if (!record.leaseId) { - const provider = providerFor(record.providerId); - record = await resumeProvision(record, provider); - return finishDestroy(record, provider); - } - return finishDestroy(record); - }); - }; - - const attachSession = async ( - request: WorkerCredentialBinding & { sessionId: string }, - ): Promise => { - if (stopping) { - throw serviceError("invalid_state", "Worker environment service is stopping"); - } - return withLock(request.environmentId, async () => { - if (stopping) { - throw serviceError("invalid_state", "Worker environment service is stopping"); - } - const current = store.get(request.environmentId); - if (!current) { - throw serviceError( - "environment_not_found", - `Unknown worker environment: ${request.environmentId}`, - ); - } - if (current.state !== "ready" && current.state !== "idle") { - throw serviceError("invalid_state", `Cannot attach worker in state: ${current.state}`); - } - let currentBuild: WorkerInstallationArtifact; - try { - currentBuild = await options.prepareInstallation("bundle"); - } catch { - throw serviceError("invalid_state", "Current worker build identity is unavailable"); - } - if ( - !current.bootstrapReceipt || - !verifyWorkerAdmissionHandshake(current.bootstrapReceipt, currentBuild) - ) { - throw serviceError( - "invalid_state", - "Worker must bootstrap the current build before attach", - ); - } - const material = credentialMaterial(); - let attached: WorkerEnvironmentRecord; - try { - attached = store.transition({ - environmentId: request.environmentId, - from: current.state, - to: "attached", - expectedOwnerEpoch: request.ownerEpoch, - patch: { - attachedSessionIds: [request.sessionId], - credential: { - credentialHash: material.credentialHash, - sessionId: request.sessionId, - rpcSetVersion: WORKER_RPC_SET_VERSION, - expiresAtMs: credentialExpiry(), - }, - }, - }); - } catch (error) { - if (error instanceof WorkerSessionAlreadyAttachedError) { - throw serviceError("invalid_state", error.message); - } - throw error; - } - if (options.liveEvents) { - let liveSessionBound: boolean; - try { - liveSessionBound = options.liveEvents.bindSession({ - environmentId: attached.environmentId, - runEpoch: attached.ownerEpoch, - sessionId: request.sessionId, - }); - } catch { - liveSessionBound = false; - } - if (!liveSessionBound) { - move(attached, "idle"); - // Preserve the bounded attachment error after rollback fences the old worker. - await tunnels?.stop(request.environmentId, current.ownerEpoch).catch(() => undefined); - throw serviceError("invalid_state", "Attached session target is unavailable"); - } - } - pendingCredentials.delete(request.environmentId); - await tunnels?.stop(request.environmentId, current.ownerEpoch); - return stageCredential( - grantFrom({ - credential: material.credential, - record: store.getCredential(request.environmentId), - }), - ); - }); - }; - - const startTunnel = async (request: WorkerTunnelRequest): Promise => { - if (stopping) { - throw serviceError("invalid_state", "Worker environment service is stopping"); - } - if (!tunnels) { - throw serviceError("invalid_state", "Worker tunnel runtime is unavailable"); - } - let startup: Promise | undefined; - await withLock(request.environmentId, async () => { - if (stopping) { - throw serviceError("invalid_state", "Worker environment service is stopping"); - } - const record = store.get(request.environmentId); - if (!record) { - throw serviceError( - "environment_not_found", - `Unknown worker environment: ${request.environmentId}`, - ); - } - if ( - !inState(record, "ready", "idle", "attached") || - record.destroyRequestedAtMs !== null || - !record.leaseId || - !record.sshEndpoint || - !record.bootstrapReceipt - ) { - throw serviceError("invalid_state", `Cannot start tunnel in state: ${record.state}`); - } - if (record.sharedHost === null) { - throw serviceError( - "provider_failure", - "Worker lease isolation is not reconciled; retry after provider inspection", - ); - } - const credential = store.getCredential(request.environmentId); - if ( - !credential || - credential.ownerEpoch !== request.ownerEpoch || - credential.expiresAtMs <= now() - ) { - throw serviceError("invalid_state", "Worker tunnel owner credential is not current"); - } - const gateway = options.resolveWorkerGateway?.(); - if (!gateway) { - throw serviceError("invalid_state", "Worker gateway ingress is unavailable"); - } - const provider = providerFor(record.providerId); - // Tunnel ownership is registered synchronously by the manager. Release the durable-state - // lock while SSH connects so drain/destroy can fence an indefinitely reconnecting start. - startup = tunnels.start({ - ...request, - bundleHash: record.bootstrapReceipt.bundleHash, - gateway, - ssh: record.sshEndpoint, - sharedHost: record.sharedHost, - resolveIdentity: identityResolverFor(record, provider, record.leaseId), - }); - }); - if (!startup) { - throw serviceError("invalid_state", "Worker tunnel failed to start"); - } - const timeoutError = serviceError( - "provider_failure", - "Worker tunnel did not connect within 3 minutes; check worker SSH reachability and retry", - ); - try { - return await withTimeout(startup, TUNNEL_START_TIMEOUT_MS, { - createError: () => timeoutError, - }); - } catch (error) { - if (error !== timeoutError) { - throw error; - } - // Stop can itself block on an unkillable SSH child; detach it (rejection observed, - // entry stays manager-tracked) so the deadline error is returned on time. Epoch-fenced - // so a stale timed-out attempt can never tear down a newer owner's tunnel. - void tunnels.stop(request.environmentId, request.ownerEpoch).catch(() => undefined); - throw timeoutError; - } - }; - - const observeDesktop = async (request: { - environmentId: string; - control: boolean; - }): Promise => { - if (options.getConfig().cloudWorkers?.desktop !== true) { - throw serviceError( - "invalid_state", - "worker desktop observe is disabled; enable the Desktop lab in Control UI Settings -> Labs (config: cloudWorkers.desktop)", - ); - } - if (stopping) { - throw serviceError("invalid_state", "Worker environment service is stopping"); - } - if (!tunnels) { - throw serviceError("invalid_state", "Worker tunnel runtime is unavailable"); - } - let startup: Promise<{ localSocketPath: string; vncPassword?: string }> | undefined; - let ownerEpoch: number | undefined; - await withLock(request.environmentId, async () => { - if (stopping) { - throw serviceError("invalid_state", "Worker environment service is stopping"); - } - const record = store.get(request.environmentId); - if (!record) { - throw serviceError( - "environment_not_found", - `Unknown worker environment: ${request.environmentId}`, - ); - } - if ( - !inState(record, "ready", "idle", "attached") || - record.destroyRequestedAtMs !== null || - !record.leaseId || - !record.sshEndpoint || - !record.desktop - ) { - throw serviceError( - "invalid_state", - "environment has no desktop; desktop is a warm-time capability of the profile", - ); - } - const provider = providerFor(record.providerId); - ownerEpoch = record.ownerEpoch; - startup = tunnels.desktop.acquire({ - environmentId: record.environmentId, - ownerEpoch: record.ownerEpoch, - ssh: record.sshEndpoint, - desktop: record.desktop, - resolveIdentity: identityResolverFor(record, provider, record.leaseId), - }); - }); - if (!startup || ownerEpoch === undefined) { - throw serviceError("invalid_state", "Worker desktop tunnel failed to start"); - } - const acquired = await startup; - const { WORKER_DESKTOP_OBSERVE_PATH, mintWorkerDesktopObserverToken } = - await import("./desktop-observe.js"); - const minted = mintWorkerDesktopObserverToken({ - environmentId: request.environmentId, - ownerEpoch, - control: request.control, - localSocketPath: acquired.localSocketPath, - nowMs: now(), - }); - return { - transport: "rfb", - wsPath: `${WORKER_DESKTOP_OBSERVE_PATH}?token=${minted.token}`, - expiresAtMs: minted.expiresAtMs, - control: request.control, - ...(acquired.vncPassword ? { vncPassword: acquired.vncPassword } : {}), - }; - }; - - const launchDesktopApp = async (request: { - environmentId: string; - app: "browser" | "terminal"; - }): Promise => { - if (options.getConfig().cloudWorkers?.desktop !== true) { - throw serviceError( - "invalid_state", - "worker desktop launch is disabled; enable the Desktop lab in Control UI Settings -> Labs (config: cloudWorkers.desktop)", - ); - } - if (stopping) { - throw serviceError("invalid_state", "Worker environment service is stopping"); - } - if (!tunnels) { - throw serviceError("invalid_state", "Worker tunnel runtime is unavailable"); - } - const requireLaunchable = () => { - if (stopping) { - throw serviceError("invalid_state", "Worker environment service is stopping"); - } - const record = store.get(request.environmentId); - if (!record) { - throw serviceError( - "environment_not_found", - `Unknown worker environment: ${request.environmentId}`, - ); - } - if ( - !inState(record, "ready", "idle", "attached") || - record.destroyRequestedAtMs !== null || - !record.leaseId || - !record.sshEndpoint || - !record.desktop - ) { - throw serviceError( - "invalid_state", - "environment has no desktop; desktop is a warm-time capability of the profile", - ); - } - const app = record.desktop.apps?.find((candidate) => candidate.id === request.app); - if (!app) { - throw serviceError( - "desktop_app_not_found", - `environment does not advertise desktop app: ${request.app}`, - ); - } - return { app, record }; - }; - - let startup: Promise | undefined; - let launchEpoch: number | undefined; - await withLock(request.environmentId, async () => { - const { app, record } = requireLaunchable(); - const provider = providerFor(record.providerId); - launchEpoch = record.ownerEpoch; - startup = tunnels.desktop.launchApp({ - environmentId: record.environmentId, - ownerEpoch: record.ownerEpoch, - ssh: record.sshEndpoint, - app, - resolveIdentity: identityResolverFor(record, provider, record.leaseId), - }); - }); - if (!startup || launchEpoch === undefined) { - throw serviceError("launcher_failure", "Worker desktop app launcher failed to start"); - } - try { - await startup; - } catch (error) { - if ( - error && - typeof error === "object" && - "code" in error && - error.code === "unsupported_platform" - ) { - throw serviceError( - "unsupported_platform", - "desktop app launch is not supported on Windows gateway hosts", - ); - } - // A teardown aborts the SSH child before mutating the durable row. Wait for the - // environment lock, then report the authoritative lifecycle state instead of a launch error. - await withLock(request.environmentId, async () => { - const { record } = requireLaunchable(); - if (record.ownerEpoch !== launchEpoch) { - throw serviceError("invalid_state", "Worker desktop app launch owner changed"); - } - }); - throw serviceError( - "launcher_failure", - `worker desktop ${request.app} launcher failed; verify the app is installed and retry`, - ); - } - await withLock(request.environmentId, async () => { - const { record } = requireLaunchable(); - if (record.ownerEpoch !== launchEpoch) { - throw serviceError("invalid_state", "Worker desktop app launch owner changed"); - } - }); - return { app: request.app, status: "ready" }; - }; - - const stopTunnel = async (environmentId: string, ownerEpoch?: number): Promise => { - await withLock(environmentId, async () => { - await tunnels?.stop(environmentId, ownerEpoch); - }); - }; + const turnRpc = createWorkerTurnRpc({ + store, + prepareInstallation: options.prepareInstallation, + applyTranscriptCommit: options.applyTranscriptCommit, + liveEvents: options.liveEvents, + placementStore: options.placementStore, + executeSessionTool: options.executeSessionTool, + inference, + isStopping: () => stopping, + now, + withLock, + }); const reconcilePass = async () => { const tasks = store.listForReconcile().map( @@ -1473,11 +310,13 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService if (!current || inState(current, "destroyed", "failed")) { return; } - await reconcileRecord(current).catch(() => - warn( - `Worker environment reconcile failed (${current.environmentId}, ${current.providerId})`, - ), - ); + await providerLifecycle + .reconcileRecord(current) + .catch(() => + warn( + `Worker environment reconcile failed (${current.environmentId}, ${current.providerId})`, + ), + ); }), ); await runTasksWithConcurrency({ tasks, limit: 8 }); @@ -1518,9 +357,9 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService unsubscribeSessionIdentityMutation?.(); unsubscribeSessionIdentityMutation = undefined; await inference.stop(); - pendingCredentials.clear(); + credentialBroker.clear(); options.liveEvents?.clear(); - await tunnels?.stopAll(); + await environmentAccess.stopAllTunnels(); const reconciliation = reconcileInFlight; if (reconciliation) { await Promise.allSettled([reconciliation]); @@ -1528,500 +367,52 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService while (activeOperations.size > 0) { await Promise.allSettled(activeOperations); } - pendingCredentials.clear(); - observedAckCursors.clear(); - pendingTerminalTurnFences.clear(); - terminalTurnFences.clear(); + credentialBroker.clear(); + turnRpc.clear(); options.liveEvents?.clear(); }; - const readPendingCredential = (binding: WorkerCredentialBinding) => { - if (stopping) { - return undefined; - } - const grant = pendingCredentials.get(binding.environmentId); - if ( - !grant || - grant.ownerEpoch !== binding.ownerEpoch || - grant.sessionId !== binding.sessionId - ) { - return undefined; - } - const environment = store.get(binding.environmentId); - const credential = store.getCredential(binding.environmentId); - const credentialHash = grant.deliveryId; - const checkedAtMs = now(); - if ( - !environment || - !inState(environment, "ready", "idle", "attached") || - environment.destroyRequestedAtMs !== null || - environment.ownerEpoch !== binding.ownerEpoch || - !credential || - credential.credentialHash !== credentialHash || - credential.ownerEpoch !== binding.ownerEpoch || - credential.sessionId !== binding.sessionId || - credential.deliveredAtMs !== null || - credential.expiresAtMs <= checkedAtMs - ) { - return undefined; - } - return { checkedAtMs, credentialHash, grant }; - }; - - const validateAttachedWorkerRequest = ( - identity: WorkerConnectionIdentity, - runEpoch: number, - request: WorkerTurnRequest, - ): - | { ok: true } - | { ok: false; closeReason: WorkerProtocolCloseReason } - | { ok: false; reason: "epoch-mismatch" | "session-not-attached" } => { - if (stopping) { - return { ok: false, closeReason: "environment-unavailable" }; - } - const placement = validateWorkerPlacement(identity); - if (!placement.valid) { - return { ok: false, closeReason: "placement-mismatch" }; - } - const turnBinding = processTurnBinding(identity); - const terminalFence = identity.sessionId - ? terminalTurnFences.get(identity.sessionId) - : undefined; - if (turnBinding && terminalFence && matchesTurnBinding(terminalFence, turnBinding)) { - const isReplay = - (request.kind === "transcript" && request.seq <= terminalFence.transcriptSeq) || - (request.kind === "live" && request.seq <= terminalFence.liveSeq); - if (!isReplay) { - return { ok: false, closeReason: "placement-mismatch" }; - } - } - const credential = store.getCredential(identity.environmentId); - if (!credential || !safeEqualSecret(credential.credentialHash, identity.credentialHash)) { - return { ok: false, closeReason: "credential-replaced" }; - } - // TTL limits unattached admission. An exact durable turn stays usable, - // including reconnects, until its terminal ACK or placement fence. - if (now() >= credential.expiresAtMs && !placement.durableClaim) { - return { ok: false, closeReason: "credential-expired" }; - } - const environment = store.get(identity.environmentId); - if (!environment || environment.destroyRequestedAtMs !== null) { - return { ok: false, closeReason: "environment-unavailable" }; - } - if ( - runEpoch !== identity.ownerEpoch || - runEpoch !== credential.ownerEpoch || - runEpoch !== environment.ownerEpoch - ) { - return { ok: false, reason: "epoch-mismatch" }; - } - if ( - environment.state !== "attached" || - !identity.sessionId || - credential.sessionId !== identity.sessionId || - environment.attachedSessionIds.length !== 1 || - environment.attachedSessionIds[0] !== identity.sessionId - ) { - return { ok: false, reason: "session-not-attached" }; - } - if (turnBinding && terminalFence && !matchesTurnBinding(terminalFence, turnBinding)) { - // Credential rotation identifies a new process turn even when a caller - // intentionally reuses its durable run id (for example, cron sessions). - terminalTurnFences.delete(turnBinding.sessionId); - } - return { ok: true }; - }; - - const commitTranscript = ( - identity: WorkerConnectionIdentity, - request: WorkerTranscriptCommitParams, - ): Promise => - withLock(identity.environmentId, async () => { - const binding = validateAttachedWorkerRequest(identity, request.runEpoch, { - kind: "transcript", - seq: request.seq, - }); - if (!binding.ok) { - return binding; - } - if (!options.applyTranscriptCommit) { - return { ok: false, closeReason: "gateway-unavailable" }; - } - const result = await options.applyTranscriptCommit({ identity, request }); - // Transcript persistence awaits outside the placement transaction. Revalidate the durable - // claim before exposing an ACK so reclamation cannot admit both owners for one session. - const currentBinding = validateAttachedWorkerRequest(identity, request.runEpoch, { - kind: "transcript", - seq: request.seq, - }); - if (!currentBinding.ok) { - return currentBinding; - } - // Stale base is a terminal sequenced outcome. Advance its durable cursor - // so the next worker commit cannot reuse the consumed sequence number. - if (result.ok || result.reason === "stale-base-leaf") { - const placement = placementBinding(identity); - const processTurn = processTurnBinding(identity); - if (!placement || !processTurn) { - return { ok: false, closeReason: "placement-mismatch" }; - } - options.placementStore?.updateAckCursors({ ...placement, transcriptSeq: request.seq }); - recordAckCursor(processTurn, { transcriptSeq: request.seq }); - } - return result; - }); - - const executeSessionTool = async ( - identity: WorkerConnectionIdentity, - toolName: WorkerSessionToolName, - request: WorkerSessionsSpawnParams | WorkerSessionsSendParams, - signal?: AbortSignal, - ): Promise => { - const validate = () => { - const requestAdmission = validateAttachedWorkerRequest(identity, identity.ownerEpoch, { - kind: "session-tool", - }); - if (!requestAdmission.ok) { - return "closeReason" in requestAdmission - ? requestAdmission - : { ok: false as const, closeReason: "placement-mismatch" as const }; - } - const binding = placementBinding(identity); - if (!binding || !options.placementStore?.isWorkerTurnToolAuthorized(binding, toolName)) { - return { ok: false as const, closeReason: "method-not-allowed" as const }; - } - return { ok: true as const }; - }; - const admitted = validate(); - if (!admitted.ok) { - return admitted; - } - if (!options.executeSessionTool) { - return { ok: false, reason: "gateway-unavailable" }; - } - let result: WorkerSessionToolResult; - try { - result = await options.executeSessionTool( - toolName === "sessions_spawn" - ? { - identity, - toolName, - request: request as WorkerSessionsSpawnParams, - ...(signal ? { signal } : {}), - } - : { - identity, - toolName, - request: request as WorkerSessionsSendParams, - ...(signal ? { signal } : {}), - }, - ); - } catch { - return { ok: false, reason: "gateway-unavailable" }; - } - // The tool may have awaited provider provisioning or another session turn. - // Never return its result after the source turn or placement was revoked. - const current = validate(); - return current.ok ? { ok: true, result } : current; - }; - - const applyLiveEvent = ( - identity: WorkerConnectionIdentity, - request: WorkerLiveEventParams, - ): WorkerLiveEventServiceResult => { - const binding = validateAttachedWorkerRequest(identity, request.runEpoch, { - kind: "live", - seq: request.seq, - }); - if (!binding.ok) { - if ("closeReason" in binding) { - return binding; - } - return { ok: false, details: { reason: binding.reason } }; - } - if (request.runId !== identity.runId) { - return { ok: false, closeReason: "placement-mismatch" }; - } - if (!options.liveEvents) { - return { ok: false, closeReason: "gateway-unavailable" }; - } - // The caller holds the environment lock, preserving order with transcript - // commits and the terminal mutation fence while this synchronous receiver runs. - const result = options.liveEvents.apply({ identity, request }); - if (result.ok) { - const placement = placementBinding(identity); - const processTurn = processTurnBinding(identity); - if (!placement || !processTurn) { - return { ok: false, closeReason: "placement-mismatch" }; - } - options.placementStore?.updateAckCursors({ - ...placement, - liveSeq: result.result.ackedSeq, - ...(isTerminalLiveEvent(request) ? { workspaceResultPending: true } : {}), - }); - recordAckCursor(processTurn, { liveSeq: result.result.ackedSeq }); - } - return result; - }; - - const pushLiveEvent = async ( - identity: WorkerConnectionIdentity, - request: WorkerLiveEventParams, - ): Promise => { - return await withLock(identity.environmentId, async () => { - const placement = placementBinding(identity); - const processTurn = processTurnBinding(identity); - const observed = processTurn ? observedAckCursorFor(processTurn) : undefined; - const wasNewSequence = request.seq > (observed?.liveSeq ?? 0); - const result = applyLiveEvent(identity, request); - if (!result.ok || !placement || !processTurn) { - return result; - } - const pending = pendingTerminalTurnFences.get(placement.sessionId); - if (pending && !matchesTurnBinding(pending, processTurn)) { - pendingTerminalTurnFences.delete(placement.sessionId); - } - if (isTerminalLiveEvent(request) && wasNewSequence) { - pendingTerminalTurnFences.set(placement.sessionId, { - ...processTurn, - terminalLiveSeq: request.seq, - }); - } - const terminal = pendingTerminalTurnFences.get(placement.sessionId); - if ( - terminal && - matchesTurnBinding(terminal, processTurn) && - result.result.ackedSeq >= terminal.terminalLiveSeq - ) { - // A gap fill can ACK a previously buffered terminal event. Fence from - // the observed high-water marks, not only from the request carrying it. - terminalTurnFences.set( - placement.sessionId, - observedAckCursorFor(processTurn) ?? - recordAckCursor(processTurn, { liveSeq: result.result.ackedSeq }), - ); - pendingTerminalTurnFences.delete(placement.sessionId); - } - return result; - }); - }; - - const revalidateInference = ( - identity: WorkerConnectionIdentity, - request: WorkerInferenceStartParams | WorkerInferenceCancelParams, - ): "epoch-mismatch" | "session-not-attached" | null => { - if (request.sessionId !== identity.sessionId) { - return "session-not-attached"; - } - const binding = validateAttachedWorkerRequest(identity, request.runEpoch, { - kind: "inference", - }); - return binding.ok ? null : "reason" in binding ? binding.reason : "session-not-attached"; - }; - - const startInference = ( - identity: WorkerConnectionIdentity, - request: WorkerInferenceStartParams, - sink: WorkerInferenceSink, - ): WorkerInferenceStartServiceResult => { - if (request.sessionId !== identity.sessionId || request.runId !== identity.runId) { - return { ok: false, reason: "session-not-attached" }; - } - const binding = validateAttachedWorkerRequest(identity, request.runEpoch, { - kind: "inference", - }); - if (!binding.ok) { - return binding; - } - return inference.start({ - identity, - request, - sink, - revalidate: () => revalidateInference(identity, request), - }); - }; - - const cancelInference = ( - identity: WorkerConnectionIdentity, - request: WorkerInferenceCancelParams, - ): WorkerInferenceCancelServiceResult => { - if (request.sessionId !== identity.sessionId || request.runId !== identity.runId) { - return { ok: false, reason: "session-not-attached" }; - } - const binding = validateAttachedWorkerRequest(identity, request.runEpoch, { - kind: "inference", - }); - if (!binding.ok) { - return binding; - } - return inference.cancel({ - identity, - request, - revalidate: () => revalidateInference(identity, request), - }); - }; - const service = { - list: () => store.list().map(project), - get: (environmentId: string) => { - const record = store.get(environmentId); - return record ? project(record) : undefined; - }, + list: environmentAccess.list, + get: environmentAccess.get, create: async (profileId: string, idempotencyKey: string) => - project(await create(profileId, idempotencyKey)), + environmentAccess.project( + await providerLifecycle.createWithProfile(profileId, idempotencyKey), + ), createFromProfileSnapshot: async ( profile: { profileId: string; providerId: string; profileSnapshot: WorkerProfile }, idempotencyKey: string, ) => - project( - await createWithProfile(profile.profileId, idempotencyKey, { + environmentAccess.project( + await providerLifecycle.createWithProfile(profile.profileId, idempotencyKey, { providerId: profile.providerId, profileSnapshot: profile.profileSnapshot, }), ), - destroy: async (environmentId: string) => project(await destroy(environmentId)), + destroy: async (environmentId: string) => + environmentAccess.project(await providerLifecycle.destroy(environmentId)), destroyUnattached: async (environmentId: string) => - project(await destroy(environmentId, { requireUnattached: true })), - observeDesktop, - launchDesktopApp, - admitWorker: async (admission: WorkerConnectParams["admission"]) => { - if (stopping) { - return { ok: false, reason: "environment-unavailable" } as const; - } - const preflightAtMs = now(); - const preflight = admitWorkerConnection({ - store, - admission, - expectedBuild: admission.handshake, - nowMs: preflightAtMs, - allowExpiredCredential: true, - }); - if (!preflight.ok) { - return preflight; - } - if (preflightAtMs >= preflight.identity.credentialExpiresAtMs) { - const placement = placementBinding(preflight.identity); - if (!placement || !options.placementStore?.validateWorkerTurn(placement)) { - return { ok: false, reason: "credential-expired" } as const; - } - } - let expectedBuild: ExpectedWorkerBuild; - try { - expectedBuild = await options.prepareInstallation("bundle"); - } catch { - return { ok: false, reason: "environment-unavailable" } as const; - } - if (stopping) { - return { ok: false, reason: "environment-unavailable" } as const; - } - const admittedAtMs = now(); - const admitted = admitWorkerConnection({ - store, - admission, - expectedBuild, - nowMs: admittedAtMs, - allowExpiredCredential: true, - }); - if (!admitted.ok) { - return admitted; - } - const expired = admittedAtMs >= admitted.identity.credentialExpiresAtMs; - if ( - !options.placementStore || - (admitted.identity.sessionId === null && admitted.identity.runId === null) - ) { - return expired ? ({ ok: false, reason: "credential-expired" } as const) : admitted; - } - const placement = placementBinding(admitted.identity); - if (!placement || !options.placementStore.validateWorkerTurn(placement)) { - return { - ok: false, - reason: expired ? "credential-expired" : "placement-mismatch", - } as const; - } - return admitted; - }, - validateWorkerConnection: (identity: WorkerConnectionIdentity) => { - if (stopping) { - return "environment-unavailable" as const; - } - const placement = validateWorkerPlacement(identity); - if (!placement.valid) { - return "placement-mismatch" as const; - } - const environmentFailure = validateWorkerConnectionIdentity({ - store, - identity, - nowMs: now(), - }); - if ( - environmentFailure && - !(environmentFailure === "credential-expired" && placement.durableClaim) - ) { - return environmentFailure; - } - return null; - }, - commitTranscript, - pushLiveEvent, - executeSessionTool, - startInference, - cancelInference, - cancelInferenceForSession: (params: { sessionId: string; runId?: string }): string[] => - inference.cancelSession(params.sessionId, params.runId), - hasInferenceForSession: (sessionId: string, runId?: string): boolean => - inference.hasSession(sessionId, runId), - resolveInferenceSessionForRunId: (runId: string): string | undefined => - inference.resolveSessionIdForRunId(runId), - attachSession, - takeMintedCredential: (binding: WorkerCredentialBinding) => - readPendingCredential(binding)?.grant, - acquireTurnCredential: (binding: WorkerCredentialBinding & { sessionId: string }) => - withLock(binding.environmentId, async () => { - const pending = readPendingCredential(binding)?.grant; - if (pending) { - return pending; - } - const environment = store.get(binding.environmentId); - if ( - !environment || - environment.state !== "attached" || - environment.ownerEpoch !== binding.ownerEpoch || - environment.attachedSessionIds.length !== 1 || - environment.attachedSessionIds[0] !== binding.sessionId - ) { - throw serviceError("invalid_state", "Worker session credential owner is not attached"); - } - const previous = store.getCredential(binding.environmentId); - const minted = mintCredentialLocked(binding); - const grant = stageCredential(minted.grant); - if (previous?.sessionId === binding.sessionId) { - options.liveEvents?.rotateCredential({ - credentialHash: minted.credentialHash, - environmentId: binding.environmentId, - newProcessTurn: true, - previousCredentialHash: previous.credentialHash, - runEpoch: binding.ownerEpoch, - sessionId: binding.sessionId, - }); - } - return grant; - }), - acknowledgeCredentialDelivery: (claim: WorkerCredentialDeliveryClaim): boolean => { - const pending = readPendingCredential(claim); - if (!pending || pending.grant.deliveryId !== claim.deliveryId) { - return false; - } - store.markCredentialDelivered({ - ...claim, - credentialHash: pending.credentialHash, - deliveredAtMs: pending.checkedAtMs, - }); - pendingCredentials.delete(claim.environmentId); - return true; - }, - startTunnel, - stopTunnel, + environmentAccess.project( + await providerLifecycle.destroy(environmentId, { requireUnattached: true }), + ), + observeDesktop: environmentAccess.observeDesktop, + launchDesktopApp: environmentAccess.launchDesktopApp, + admitWorker: turnRpc.admitWorker, + validateWorkerConnection: turnRpc.validateWorkerConnection, + commitTranscript: turnRpc.commitTranscript, + pushLiveEvent: turnRpc.pushLiveEvent, + executeSessionTool: turnRpc.executeSessionTool, + startInference: turnRpc.startInference, + cancelInference: turnRpc.cancelInference, + cancelInferenceForSession: turnRpc.cancelInferenceForSession, + hasInferenceForSession: turnRpc.hasInferenceForSession, + resolveInferenceSessionForRunId: turnRpc.resolveInferenceSessionForRunId, + attachSession: credentialBroker.attachSession, + takeMintedCredential: credentialBroker.takeMintedCredential, + acquireTurnCredential: credentialBroker.acquireTurnCredential, + acknowledgeCredentialDelivery: credentialBroker.acknowledgeCredentialDelivery, + startTunnel: environmentAccess.startTunnel, + stopTunnel: environmentAccess.stopTunnel, reconcileOnce, start, stop, @@ -2033,4 +424,3 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService } export type WorkerEnvironmentService = ReturnType; -/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ diff --git a/src/gateway/worker-environments/worker-turn-rpc.test.ts b/src/gateway/worker-environments/worker-turn-rpc.test.ts new file mode 100644 index 000000000000..19c03a97915e --- /dev/null +++ b/src/gateway/worker-environments/worker-turn-rpc.test.ts @@ -0,0 +1,563 @@ +import { describe, expect, it, vi } from "vitest"; +import { hashWorkerCredential } from "./credential.js"; +import * as support from "./service.test-support.js"; + +type WorkerEnvironmentServiceOptions = support.WorkerEnvironmentServiceOptions; + +describe("worker environment service", () => { + support.setupWorkerEnvironmentServiceSuite(); + + it("admits an npm-installed worker from canonical bundle identity without registry access", async () => { + const environmentId = "worker-npm-admission"; + support.seedReady(environmentId, "npm"); + support.testState.prepareInstallation = vi.fn(async (install) => { + if (install === "npm") { + throw new Error("registry unavailable"); + } + return support.BUNDLE_ARTIFACT; + }); + const workerService = support.createService(support.createProvider()); + + await expect( + workerService.admitWorker(support.admissionFor(environmentId)), + ).resolves.toMatchObject({ + ok: true, + }); + expect(support.testState.prepareInstallation).toHaveBeenCalledTimes(1); + expect(support.testState.prepareInstallation).toHaveBeenCalledWith("bundle"); + }); + + it("fences transcript commits by current epoch and exact session credential binding", async () => { + const environmentId = "worker-transcript-fence"; + const sessionId = "session-transcript-fence"; + const identity = support.seedAttachedIdentity(environmentId, sessionId); + const applyTranscriptCommit = support.successfulTranscriptCommit("entry-1"); + const workerService = support.createService(support.createProvider(), { + applyTranscriptCommit, + }); + const request = support.transcriptRequest(identity, "hello"); + + await expect(workerService.commitTranscript(identity, request)).resolves.toMatchObject({ + ok: true, + }); + expect(applyTranscriptCommit).toHaveBeenCalledOnce(); + + await expect( + workerService.commitTranscript(identity, { + ...request, + runEpoch: identity.ownerEpoch + 1, + seq: 2, + }), + ).resolves.toEqual({ ok: false, reason: "epoch-mismatch" }); + support.testState.stateDb.db + .prepare("UPDATE worker_environment_credentials SET session_id = ? WHERE environment_id = ?") + .run("session-other", environmentId); + await expect(workerService.commitTranscript(identity, { ...request, seq: 2 })).resolves.toEqual( + { ok: false, reason: "session-not-attached" }, + ); + expect(applyTranscriptCommit).toHaveBeenCalledOnce(); + }); + + it("admits only a gateway-preclaimed worker placement and fences later requests", async () => { + const environmentId = "worker-placement-fence"; + const sessionId = "session-placement-fence"; + const { identity, placementStore, workerService } = support.placementHarness( + environmentId, + sessionId, + ); + const admission = { + environmentId, + credential: [support.CREDENTIAL, environmentId, sessionId].join("-"), + sessionId, + runId: "run-1", + ownerEpoch: identity.ownerEpoch, + rpcSetVersion: 1, + handshake: support.BOOTSTRAP_RECEIPT, + }; + + await expect(workerService.admitWorker(admission)).resolves.toMatchObject({ ok: true }); + await expect(workerService.admitWorker(admission)).resolves.toMatchObject({ ok: true }); + expect(placementStore.validateWorkerTurn).toHaveBeenLastCalledWith({ + sessionId, + environmentId, + ownerEpoch: identity.ownerEpoch, + runId: "run-1", + }); + expect(placementStore.validateWorkerTurn).toHaveBeenCalledTimes(2); + expect(workerService.validateWorkerConnection(identity)).toBeNull(); + + const warmEnvironmentId = "worker-placement-warm"; + support.seedReady(warmEnvironmentId); + const warmAdmission = await workerService.admitWorker(support.admissionFor(warmEnvironmentId)); + expect(warmAdmission).toMatchObject({ ok: true }); + if (!warmAdmission.ok) { + throw new Error("warm worker admission failed"); + } + expect(workerService.validateWorkerConnection(warmAdmission.identity)).toBeNull(); + expect(placementStore.validateWorkerTurn).toHaveBeenCalledTimes(3); + + placementStore.validateWorkerTurn.mockReturnValue(false); + await expect( + workerService.admitWorker({ ...admission, runId: "run-conflict" }), + ).resolves.toEqual({ ok: false, reason: "placement-mismatch" }); + + placementStore.validateWorkerTurn.mockReturnValue(true); + support.testState.nowMs += 10_000; + expect(workerService.validateWorkerConnection(identity)).toBeNull(); + expect(workerService.validateWorkerConnection(warmAdmission.identity)).toBe( + "credential-expired", + ); + await expect(workerService.admitWorker(admission)).resolves.toMatchObject({ + ok: true, + identity: { sessionId, runId: "run-1" }, + }); + + placementStore.validateWorkerTurn.mockReturnValue(false); + expect(workerService.validateWorkerConnection(identity)).toBe("placement-mismatch"); + vi.mocked(support.testState.prepareInstallation).mockClear(); + await expect(workerService.admitWorker(admission)).resolves.toEqual({ + ok: false, + reason: "credential-expired", + }); + expect(support.testState.prepareInstallation).not.toHaveBeenCalled(); + await expect( + workerService.commitTranscript(identity, support.transcriptRequest(identity, "fenced")), + ).resolves.toEqual({ ok: false, closeReason: "placement-mismatch" }); + }); + + it("does not rotate an expired delivered credential while its durable turn is active", async () => { + const environmentId = "worker-expired-active-turn"; + const sessionId = "session-expired-active-turn"; + const liveEvents = support.createLiveEvents(); + const { identity, workerService } = support.placementHarness(environmentId, sessionId, { + liveEvents, + }); + support.testState.store.markCredentialDelivered({ + environmentId, + credentialHash: identity.credentialHash, + ownerEpoch: identity.ownerEpoch, + sessionId, + deliveredAtMs: support.testState.nowMs, + }); + support.testState.nowMs = identity.credentialExpiresAtMs; + + await workerService.reconcileOnce(); + + expect(support.testState.store.getCredential(environmentId)?.credentialHash).toBe( + identity.credentialHash, + ); + expect(liveEvents.rotateCredential).not.toHaveBeenCalled(); + }); + + it("persists worker transcript and terminal live ACK cursors", async () => { + const applyTranscriptCommit = support.successfulTranscriptCommit("entry-placement"); + const { liveEvents } = support.sequencedLiveEvents(); + const { identity, placementStore, workerService } = support.placementHarness( + "worker-placement-ack", + "session-placement-ack", + { + applyTranscriptCommit, + liveEvents, + }, + ); + const binding = support.placementBinding(identity); + + await expect( + workerService.commitTranscript( + identity, + support.transcriptRequest(identity, "commit", { seq: 7 }), + ), + ).resolves.toMatchObject({ ok: true }); + expect(placementStore.updateAckCursors).toHaveBeenCalledWith({ + ...binding, + transcriptSeq: 7, + }); + + await expect( + workerService.pushLiveEvent(identity, support.terminalEvent(identity)), + ).resolves.toEqual({ + ok: true, + result: { ackedSeq: 1 }, + }); + expect(placementStore.updateAckCursors).toHaveBeenLastCalledWith({ + ...binding, + liveSeq: 1, + workspaceResultPending: true, + }); + }); + + it("uses worker finishing as the durable workspace-result fence", async () => { + const { liveEvents } = support.sequencedLiveEvents(); + const { identity, placementStore, workerService } = support.placementHarness( + "worker-placement-finishing", + "session-placement-finishing", + { liveEvents }, + ); + const terminal = support.terminalEvent(identity); + const finishing = { + ...terminal, + event: { + kind: "lifecycle" as const, + payload: { phase: "finishing" as const, startedAt: 1, endedAt: 2 }, + }, + }; + + await expect(workerService.pushLiveEvent(identity, finishing)).resolves.toEqual({ + ok: true, + result: { ackedSeq: 1 }, + }); + expect(placementStore.updateAckCursors).toHaveBeenLastCalledWith({ + ...support.placementBinding(identity), + liveSeq: 1, + workspaceResultPending: true, + }); + }); + + it("does not ACK a transcript commit after its worker claim is fenced", async () => { + let finishCommit: (() => void) | undefined; + const commitBlocked = new Promise((resolve) => { + finishCommit = resolve; + }); + const applyTranscriptCommit = support.successfulTranscriptCommit( + "entry-placement-race", + () => commitBlocked, + ); + const { identity, placementStore, workerService } = support.placementHarness( + "worker-placement-race", + "session-placement-race", + { applyTranscriptCommit }, + ); + + const commit = workerService.commitTranscript( + identity, + support.transcriptRequest(identity, "commit before claim fence"), + ); + await support.waitForFast(() => expect(applyTranscriptCommit).toHaveBeenCalledOnce()); + placementStore.validateWorkerTurn.mockReturnValue(false); + finishCommit?.(); + + await expect(commit).resolves.toEqual({ ok: false, closeReason: "placement-mismatch" }); + expect(placementStore.validateWorkerTurn).toHaveBeenCalledTimes(2); + expect(placementStore.updateAckCursors).not.toHaveBeenCalled(); + }); + + it("advances the transcript cursor when a stale-base commit consumes its sequence", async () => { + const applyTranscriptCommit = vi + .fn>() + .mockResolvedValueOnce({ ok: false, reason: "stale-base-leaf" }) + .mockResolvedValueOnce({ ok: false, reason: "invalid-batch" }); + const { identity, placementStore, workerService } = support.placementHarness( + "worker-placement-stale", + "session-placement-stale", + { applyTranscriptCommit }, + ); + const request = support.transcriptRequest(identity, "stale commit", { + seq: 11, + baseLeafId: "stale-leaf", + }); + + await expect(workerService.commitTranscript(identity, request)).resolves.toEqual({ + ok: false, + reason: "stale-base-leaf", + }); + expect(placementStore.updateAckCursors).toHaveBeenCalledWith({ + ...support.placementBinding(identity), + transcriptSeq: 11, + }); + + await expect( + workerService.commitTranscript(identity, { ...request, seq: 12 }), + ).resolves.toEqual({ ok: false, reason: "invalid-batch" }); + expect(placementStore.updateAckCursors).toHaveBeenCalledOnce(); + }); + + it("fences after a buffered terminal event becomes acknowledged by a gap fill", async () => { + const applyTranscriptCommit = support.successfulTranscriptCommit("entry-after-terminal-gap"); + const { apply: liveApply, liveEvents } = support.sequencedLiveEvents((seq) => + seq === 1 ? 2 : 0, + ); + const { identity, placementStore, workerService } = support.placementHarness( + "worker-placement-gap", + "session-placement-gap", + { + applyTranscriptCommit, + liveEvents, + }, + ); + + await expect( + workerService.pushLiveEvent(identity, support.terminalEvent(identity, { seq: 2 })), + ).resolves.toEqual({ ok: true, result: { ackedSeq: 0 } }); + expect(placementStore.updateAckCursors).toHaveBeenCalledWith({ + ...support.placementBinding(identity), + liveSeq: 0, + workspaceResultPending: true, + }); + + await expect( + workerService.pushLiveEvent(identity, support.assistantEvent(identity, "fills gap")), + ).resolves.toEqual({ ok: true, result: { ackedSeq: 2 } }); + await expect( + workerService.commitTranscript( + identity, + support.transcriptRequest(identity, "late transcript"), + ), + ).resolves.toEqual({ ok: false, closeReason: "placement-mismatch" }); + await expect( + workerService.pushLiveEvent( + identity, + support.assistantEvent(identity, "late", { lastAckedSeq: 2, seq: 3 }), + ), + ).resolves.toEqual({ ok: false, closeReason: "placement-mismatch" }); + expect(applyTranscriptCommit).not.toHaveBeenCalled(); + expect(liveApply).toHaveBeenCalledTimes(2); + }); + + it("applies a terminal ACK only after its transcript commit finishes", async () => { + let finishCommit: (() => void) | undefined; + const commitBlocked = new Promise((resolve) => { + finishCommit = resolve; + }); + const applyTranscriptCommit = support.successfulTranscriptCommit( + "entry-order", + () => commitBlocked, + ); + const { liveEvents } = support.sequencedLiveEvents(); + const { identity, placementStore, workerService } = support.placementHarness( + "worker-placement-order", + "session-placement-order", + { applyTranscriptCommit, liveEvents }, + ); + + const commit = workerService.commitTranscript( + identity, + support.transcriptRequest(identity, "commit before terminal"), + ); + await support.waitForFast(() => expect(applyTranscriptCommit).toHaveBeenCalledOnce()); + const terminal = workerService.pushLiveEvent(identity, support.terminalEvent(identity)); + await Promise.resolve(); + expect(placementStore.updateAckCursors).not.toHaveBeenCalled(); + + finishCommit?.(); + await expect(commit).resolves.toMatchObject({ ok: true }); + await expect(terminal).resolves.toEqual({ ok: true, result: { ackedSeq: 1 } }); + expect(placementStore.updateAckCursors.mock.calls).toEqual([ + [ + { + ...support.placementBinding(identity), + transcriptSeq: 1, + }, + ], + [ + { + ...support.placementBinding(identity), + liveSeq: 1, + workspaceResultPending: true, + }, + ], + ]); + }); + + it("fences post-terminal mutations while preserving sequenced replays", async () => { + const applyTranscriptCommit = support.successfulTranscriptCommit("entry-terminal"); + const { apply: liveApply, liveEvents } = support.sequencedLiveEvents(); + const executeInference = vi.fn( + async () => ({ + type: "error", + reason: "provider-error", + message: "Provider request failed", + }), + ); + const { identity, workerService } = support.placementHarness( + "worker-terminal-fence", + "session-terminal-fence", + { applyTranscriptCommit, executeInference, liveEvents }, + ); + const transcript = support.transcriptRequest(identity, "terminal fence"); + const terminal = support.terminalEvent(identity); + + await expect(workerService.commitTranscript(identity, transcript)).resolves.toMatchObject({ + ok: true, + }); + await expect(workerService.pushLiveEvent(identity, terminal)).resolves.toEqual({ + ok: true, + result: { ackedSeq: 1 }, + }); + + await expect(workerService.commitTranscript(identity, transcript)).resolves.toMatchObject({ + ok: true, + }); + await expect( + workerService.commitTranscript(identity, { ...transcript, seq: 2 }), + ).resolves.toEqual({ ok: false, closeReason: "placement-mismatch" }); + expect(applyTranscriptCommit).toHaveBeenCalledTimes(2); + + await expect(workerService.pushLiveEvent(identity, terminal)).resolves.toEqual({ + ok: true, + result: { ackedSeq: 1 }, + }); + await expect( + workerService.pushLiveEvent(identity, support.assistantEvent(identity, "late", { seq: 2 })), + ).resolves.toEqual({ ok: false, closeReason: "placement-mismatch" }); + expect(liveApply).toHaveBeenCalledTimes(2); + + expect( + workerService.startInference(identity, support.inferenceRequest(identity), { + connectionId: "connection-terminal-fence", + send: vi.fn(), + }), + ).toEqual({ ok: false, closeReason: "placement-mismatch" }); + expect(workerService.cancelInference(identity, support.inferenceRequest(identity))).toEqual({ + ok: false, + closeReason: "placement-mismatch", + }); + expect(executeInference).not.toHaveBeenCalled(); + + const rotatedCredentialHash = hashWorkerCredential( + ["rotated", identity.environmentId, identity.sessionId].join("-"), + ); + support.testState.stateDb.db + .prepare( + "UPDATE worker_environment_credentials SET credential_hash = ? WHERE environment_id = ?", + ) + .run(rotatedCredentialHash, identity.environmentId); + const rotatedIdentity = { ...identity, credentialHash: rotatedCredentialHash }; + await expect( + workerService.commitTranscript(rotatedIdentity, { ...transcript, seq: 2 }), + ).resolves.toMatchObject({ ok: true }); + expect(applyTranscriptCommit).toHaveBeenCalledTimes(3); + }); + + it("does not treat a terminal event on an already ACKed sequence as authoritative", async () => { + const applyTranscriptCommit = support.successfulTranscriptCommit("entry-after-reuse"); + const { liveEvents } = support.sequencedLiveEvents(); + const { identity, workerService } = support.placementHarness( + "worker-terminal-reuse", + "session-terminal-reuse", + { applyTranscriptCommit, liveEvents }, + ); + const event = support.assistantEvent(identity, "first"); + + await expect(workerService.pushLiveEvent(identity, event)).resolves.toMatchObject({ ok: true }); + await expect( + workerService.pushLiveEvent(identity, support.terminalEvent(identity)), + ).resolves.toMatchObject({ ok: true }); + await expect( + workerService.commitTranscript( + identity, + support.transcriptRequest(identity, "still mutable"), + ), + ).resolves.toMatchObject({ ok: true }); + expect(applyTranscriptCommit).toHaveBeenCalledOnce(); + }); + + it("fences inference by epoch and the durable session credential", async () => { + const identity = support.seedAttachedIdentity( + "worker-inference-fence", + "session-inference-fence", + ); + const executeInference = vi.fn( + async () => ({ + type: "error", + reason: "provider-error", + message: "Provider request failed", + }), + ); + const workerService = support.createService(support.createProvider(), { executeInference }); + const request = support.inferenceRequest(identity); + expect( + workerService.startInference( + identity, + { ...request, sessionId: "session-other" }, + { connectionId: "connection-a", send: vi.fn() }, + ), + ).toEqual({ ok: false, reason: "session-not-attached" }); + expect( + workerService.startInference( + identity, + { ...request, runEpoch: request.runEpoch + 1 }, + { connectionId: "connection-b", send: vi.fn() }, + ), + ).toEqual({ ok: false, reason: "epoch-mismatch" }); + + const send = vi.fn(); + const started = workerService.startInference(identity, request, { + connectionId: "connection-c", + send, + }); + expect(started.ok).toBe(true); + if (!started.ok) { + throw new Error("inference fixture failed to start"); + } + support.testState.stateDb.db + .prepare( + "UPDATE worker_environment_credentials SET credential_hash = ? WHERE environment_id = ?", + ) + .run( + hashWorkerCredential(["replacement", identity.environmentId].join("-")), + identity.environmentId, + ); + started.launch(); + await support.waitForFast(() => expect(send).toHaveBeenCalledOnce()); + expect(executeInference).not.toHaveBeenCalled(); + expect(send.mock.calls[0]?.[0]).toMatchObject({ + event: "worker.inference.terminal", + payload: { outcome: { reason: "session-not-attached" } }, + }); + }); + + it("fences and rotates live credentials", async () => { + const environmentId = "worker-live"; + const sessionId = "session-live"; + const identity = support.seedAttachedIdentity(environmentId, sessionId); + const liveEvents = support.createLiveEvents(); + let inferenceSignal: AbortSignal | undefined; + const executeInference = vi.fn( + async ({ signal }) => { + inferenceSignal = signal; + await new Promise((resolve) => { + signal.addEventListener("abort", () => resolve(), { once: true }); + }); + return { type: "error", reason: "cancelled", message: "Inference cancelled" }; + }, + ); + const workerService = support.createService(support.createProvider(), { + executeInference, + liveEvents, + }); + const request = { ...support.LIVE_EVENT, runEpoch: identity.ownerEpoch }; + const push = workerService.pushLiveEvent.bind(workerService, identity); + await push(request); + await expect(push({ ...request, runEpoch: identity.ownerEpoch + 1 })).resolves.toEqual({ + ok: false, + details: { reason: "epoch-mismatch" }, + }); + const started = workerService.startInference(identity, support.inferenceRequest(identity), { + connectionId: "connection-rotation", + send: vi.fn(), + }); + if (!started.ok) { + throw new Error("inference fixture failed to start"); + } + started.launch(); + await support.waitForFast(() => expect(executeInference).toHaveBeenCalledOnce()); + support.testState.stateDb.db + .prepare("UPDATE worker_environment_credentials SET session_id = ? WHERE environment_id = ?") + .run("session-other", environmentId); + await expect(push({ ...request, seq: 2 })).resolves.toEqual({ + ok: false, + details: { reason: "session-not-attached" }, + }); + liveEvents.rotateCredential.mockClear(); + support.testState.nowMs += 10_000; + await workerService.reconcileOnce(); + expect(inferenceSignal?.aborted).toBe(true); + expect(liveEvents.rotateCredential).toHaveBeenCalledWith( + expect.objectContaining({ + credentialHash: support.testState.store.getCredential(environmentId)?.credentialHash, + previousCredentialHash: identity.credentialHash, + runEpoch: identity.ownerEpoch, + }), + ); + }); +}); diff --git a/src/gateway/worker-environments/worker-turn-rpc.ts b/src/gateway/worker-environments/worker-turn-rpc.ts new file mode 100644 index 000000000000..aebc40f27f73 --- /dev/null +++ b/src/gateway/worker-environments/worker-turn-rpc.ts @@ -0,0 +1,597 @@ +import type { + WorkerConnectParams, + WorkerLiveEventParams, + WorkerProtocolCloseReason, + WorkerSessionsSendParams, + WorkerSessionsSpawnParams, + WorkerSessionToolResult, + WorkerTranscriptCommitErrorReason, + WorkerTranscriptCommitParams, + WorkerTranscriptCommitResult, +} from "../../../packages/gateway-protocol/src/schema/worker-admission.js"; +import type { + WorkerInferenceCancelParams, + WorkerInferenceCancelResult, + WorkerInferenceErrorReason, + WorkerInferenceStartParams, + WorkerInferenceStartResult, +} from "../../../packages/gateway-protocol/src/schema/worker-inference.js"; +import { safeEqualSecret } from "../../security/secret-equal.js"; +import type { WorkerSessionToolName } from "../../worker/tool-authority.js"; +import { + admitWorkerConnection, + validateWorkerConnectionIdentity, + type ExpectedWorkerBuild, + type WorkerConnectionIdentity, +} from "./admission.js"; +import type { WorkerInstallationArtifact } from "./bundle.js"; +import { createWorkerInferenceManager, type WorkerInferenceSink } from "./inference.js"; +import type { WorkerLiveEventApplicationResult, WorkerLiveEventReceiver } from "./live-events.js"; +import type { + WorkerPlacementTurnBinding, + WorkerSessionPlacementGate, +} from "./placement-worker-gate.js"; +import type { WorkerEnvironmentStore } from "./store.js"; + +type WorkerProcessTurnBinding = WorkerPlacementTurnBinding & { + credentialHash: string; +}; + +type WorkerTerminalTurnFence = WorkerProcessTurnBinding & { + transcriptSeq: number; + liveSeq: number; +}; + +type WorkerPendingTerminalTurnFence = WorkerProcessTurnBinding & { + terminalLiveSeq: number; +}; + +type WorkerTurnRequest = + | { kind: "inference" } + | { kind: "live"; seq: number } + | { kind: "transcript"; seq: number } + | { kind: "session-tool" }; + +type WorkerTranscriptCommitApplicationResult = + | { ok: true; result: WorkerTranscriptCommitResult } + | { ok: false; reason: WorkerTranscriptCommitErrorReason }; + +type WorkerTranscriptCommitServiceResult = + | WorkerTranscriptCommitApplicationResult + | { ok: false; closeReason: WorkerProtocolCloseReason }; + +type WorkerLiveEventServiceResult = + | WorkerLiveEventApplicationResult + | { ok: false; closeReason: WorkerProtocolCloseReason }; + +type WorkerInferenceStartServiceResult = + | { + ok: true; + result: WorkerInferenceStartResult; + launch: () => void; + } + | { ok: false; reason: WorkerInferenceErrorReason } + | { ok: false; closeReason: WorkerProtocolCloseReason }; + +type WorkerInferenceCancelServiceResult = + | { ok: true; result: WorkerInferenceCancelResult } + | { ok: false; reason: WorkerInferenceErrorReason } + | { ok: false; closeReason: WorkerProtocolCloseReason }; + +type WorkerSessionToolServiceResult = + | { ok: true; result: WorkerSessionToolResult } + | { ok: false; closeReason: WorkerProtocolCloseReason } + | { ok: false; reason: WorkerProtocolCloseReason }; + +type WorkerTurnRpcOptions = { + store: WorkerEnvironmentStore; + prepareInstallation: ( + install: WorkerInstallationArtifact["install"], + ) => Promise; + applyTranscriptCommit?: (params: { + identity: WorkerConnectionIdentity; + request: WorkerTranscriptCommitParams; + }) => Promise; + liveEvents?: Pick; + placementStore?: WorkerSessionPlacementGate; + executeSessionTool?: ( + params: + | { + identity: WorkerConnectionIdentity; + toolName: "sessions_spawn"; + request: WorkerSessionsSpawnParams; + signal?: AbortSignal; + } + | { + identity: WorkerConnectionIdentity; + toolName: "sessions_send"; + request: WorkerSessionsSendParams; + signal?: AbortSignal; + }, + ) => Promise; + inference: ReturnType; + isStopping: () => boolean; + now: () => number; + withLock: (environmentId: string, task: () => Promise) => Promise; +}; + +export function createWorkerTurnRpc(options: WorkerTurnRpcOptions) { + const { store } = options; + const inference = options.inference; + const now = options.now; + const withLock = options.withLock; + const observedAckCursors = new Map(); + const pendingTerminalTurnFences = new Map(); + const terminalTurnFences = new Map(); + + const placementBinding = ( + identity: WorkerConnectionIdentity, + ): WorkerPlacementTurnBinding | undefined => { + if (!identity.sessionId || !identity.runId) { + return undefined; + } + return { + sessionId: identity.sessionId, + environmentId: identity.environmentId, + ownerEpoch: identity.ownerEpoch, + runId: identity.runId, + }; + }; + + const processTurnBinding = ( + identity: WorkerConnectionIdentity, + ): WorkerProcessTurnBinding | undefined => { + const placement = placementBinding(identity); + return placement ? { ...placement, credentialHash: identity.credentialHash } : undefined; + }; + + const matchesTurnBinding = ( + left: WorkerProcessTurnBinding, + right: WorkerProcessTurnBinding, + ): boolean => + left.sessionId === right.sessionId && + left.environmentId === right.environmentId && + left.ownerEpoch === right.ownerEpoch && + left.runId === right.runId && + safeEqualSecret(left.credentialHash, right.credentialHash); + + const recordAckCursor = ( + binding: WorkerProcessTurnBinding, + cursor: { transcriptSeq: number } | { liveSeq: number }, + ): WorkerTerminalTurnFence => { + const current = observedAckCursors.get(binding.sessionId); + const currentTurn = current && matchesTurnBinding(current, binding) ? current : undefined; + const next: WorkerTerminalTurnFence = { + ...binding, + transcriptSeq: + "transcriptSeq" in cursor + ? Math.max(currentTurn?.transcriptSeq ?? 0, cursor.transcriptSeq) + : (currentTurn?.transcriptSeq ?? 0), + liveSeq: + "liveSeq" in cursor + ? Math.max(currentTurn?.liveSeq ?? 0, cursor.liveSeq) + : (currentTurn?.liveSeq ?? 0), + }; + observedAckCursors.set(binding.sessionId, next); + return next; + }; + + const observedAckCursorFor = ( + binding: WorkerProcessTurnBinding, + ): WorkerTerminalTurnFence | undefined => { + const observed = observedAckCursors.get(binding.sessionId); + return observed && matchesTurnBinding(observed, binding) ? observed : undefined; + }; + + const validateWorkerPlacement = ( + identity: WorkerConnectionIdentity, + ): { durableClaim: boolean; valid: boolean } => { + if (!options.placementStore) { + return { durableClaim: false, valid: true }; + } + if (identity.sessionId === null && identity.runId === null) { + return { durableClaim: false, valid: true }; + } + const binding = placementBinding(identity); + const valid = binding ? options.placementStore.validateWorkerTurn(binding) : false; + return { durableClaim: valid, valid }; + }; + + const isTerminalLiveEvent = (request: WorkerLiveEventParams): boolean => + request.event.kind === "lifecycle" && + (request.event.payload.phase === "finishing" || + request.event.payload.phase === "end" || + (request.event.payload.phase === "error" && + (request.event.payload.aborted === true || + request.event.payload.fallbackExhaustedFailure === true))); + + const validateAttachedWorkerRequest = ( + identity: WorkerConnectionIdentity, + runEpoch: number, + request: WorkerTurnRequest, + ): + | { ok: true } + | { ok: false; closeReason: WorkerProtocolCloseReason } + | { ok: false; reason: "epoch-mismatch" | "session-not-attached" } => { + if (options.isStopping()) { + return { ok: false, closeReason: "environment-unavailable" }; + } + const placement = validateWorkerPlacement(identity); + if (!placement.valid) { + return { ok: false, closeReason: "placement-mismatch" }; + } + const turnBinding = processTurnBinding(identity); + const terminalFence = identity.sessionId + ? terminalTurnFences.get(identity.sessionId) + : undefined; + if (turnBinding && terminalFence && matchesTurnBinding(terminalFence, turnBinding)) { + const isReplay = + (request.kind === "transcript" && request.seq <= terminalFence.transcriptSeq) || + (request.kind === "live" && request.seq <= terminalFence.liveSeq); + if (!isReplay) { + return { ok: false, closeReason: "placement-mismatch" }; + } + } + const credential = store.getCredential(identity.environmentId); + if (!credential || !safeEqualSecret(credential.credentialHash, identity.credentialHash)) { + return { ok: false, closeReason: "credential-replaced" }; + } + // TTL limits unattached admission. An exact durable turn stays usable, + // including reconnects, until its terminal ACK or placement fence. + if (now() >= credential.expiresAtMs && !placement.durableClaim) { + return { ok: false, closeReason: "credential-expired" }; + } + const environment = store.get(identity.environmentId); + if (!environment || environment.destroyRequestedAtMs !== null) { + return { ok: false, closeReason: "environment-unavailable" }; + } + if ( + runEpoch !== identity.ownerEpoch || + runEpoch !== credential.ownerEpoch || + runEpoch !== environment.ownerEpoch + ) { + return { ok: false, reason: "epoch-mismatch" }; + } + if ( + environment.state !== "attached" || + !identity.sessionId || + credential.sessionId !== identity.sessionId || + environment.attachedSessionIds.length !== 1 || + environment.attachedSessionIds[0] !== identity.sessionId + ) { + return { ok: false, reason: "session-not-attached" }; + } + if (turnBinding && terminalFence && !matchesTurnBinding(terminalFence, turnBinding)) { + // Credential rotation identifies a new process turn even when a caller + // intentionally reuses its durable run id (for example, cron sessions). + terminalTurnFences.delete(turnBinding.sessionId); + } + return { ok: true }; + }; + + const commitTranscript = ( + identity: WorkerConnectionIdentity, + request: WorkerTranscriptCommitParams, + ): Promise => + withLock(identity.environmentId, async () => { + const binding = validateAttachedWorkerRequest(identity, request.runEpoch, { + kind: "transcript", + seq: request.seq, + }); + if (!binding.ok) { + return binding; + } + if (!options.applyTranscriptCommit) { + return { ok: false, closeReason: "gateway-unavailable" }; + } + const result = await options.applyTranscriptCommit({ identity, request }); + // Transcript persistence awaits outside the placement transaction. Revalidate the durable + // claim before exposing an ACK so reclamation cannot admit both owners for one session. + const currentBinding = validateAttachedWorkerRequest(identity, request.runEpoch, { + kind: "transcript", + seq: request.seq, + }); + if (!currentBinding.ok) { + return currentBinding; + } + // Stale base is a terminal sequenced outcome. Advance its durable cursor + // so the next worker commit cannot reuse the consumed sequence number. + if (result.ok || result.reason === "stale-base-leaf") { + const placement = placementBinding(identity); + const processTurn = processTurnBinding(identity); + if (!placement || !processTurn) { + return { ok: false, closeReason: "placement-mismatch" }; + } + options.placementStore?.updateAckCursors({ ...placement, transcriptSeq: request.seq }); + recordAckCursor(processTurn, { transcriptSeq: request.seq }); + } + return result; + }); + + const executeSessionTool = async ( + identity: WorkerConnectionIdentity, + toolName: WorkerSessionToolName, + request: WorkerSessionsSpawnParams | WorkerSessionsSendParams, + signal?: AbortSignal, + ): Promise => { + const validate = () => { + const requestAdmission = validateAttachedWorkerRequest(identity, identity.ownerEpoch, { + kind: "session-tool", + }); + if (!requestAdmission.ok) { + return "closeReason" in requestAdmission + ? requestAdmission + : { ok: false as const, closeReason: "placement-mismatch" as const }; + } + const binding = placementBinding(identity); + if (!binding || !options.placementStore?.isWorkerTurnToolAuthorized(binding, toolName)) { + return { ok: false as const, closeReason: "method-not-allowed" as const }; + } + return { ok: true as const }; + }; + const admitted = validate(); + if (!admitted.ok) { + return admitted; + } + if (!options.executeSessionTool) { + return { ok: false, reason: "gateway-unavailable" }; + } + let result: WorkerSessionToolResult; + try { + result = await options.executeSessionTool( + toolName === "sessions_spawn" + ? { + identity, + toolName, + request: request as WorkerSessionsSpawnParams, + ...(signal ? { signal } : {}), + } + : { + identity, + toolName, + request: request as WorkerSessionsSendParams, + ...(signal ? { signal } : {}), + }, + ); + } catch { + return { ok: false, reason: "gateway-unavailable" }; + } + // The tool may have awaited provider provisioning or another session turn. + // Never return its result after the source turn or placement was revoked. + const current = validate(); + return current.ok ? { ok: true, result } : current; + }; + + const applyLiveEvent = ( + identity: WorkerConnectionIdentity, + request: WorkerLiveEventParams, + ): WorkerLiveEventServiceResult => { + const binding = validateAttachedWorkerRequest(identity, request.runEpoch, { + kind: "live", + seq: request.seq, + }); + if (!binding.ok) { + if ("closeReason" in binding) { + return binding; + } + return { ok: false, details: { reason: binding.reason } }; + } + if (request.runId !== identity.runId) { + return { ok: false, closeReason: "placement-mismatch" }; + } + if (!options.liveEvents) { + return { ok: false, closeReason: "gateway-unavailable" }; + } + // The caller holds the environment lock, preserving order with transcript + // commits and the terminal mutation fence while this synchronous receiver runs. + const result = options.liveEvents.apply({ identity, request }); + if (result.ok) { + const placement = placementBinding(identity); + const processTurn = processTurnBinding(identity); + if (!placement || !processTurn) { + return { ok: false, closeReason: "placement-mismatch" }; + } + options.placementStore?.updateAckCursors({ + ...placement, + liveSeq: result.result.ackedSeq, + ...(isTerminalLiveEvent(request) ? { workspaceResultPending: true } : {}), + }); + recordAckCursor(processTurn, { liveSeq: result.result.ackedSeq }); + } + return result; + }; + + const pushLiveEvent = async ( + identity: WorkerConnectionIdentity, + request: WorkerLiveEventParams, + ): Promise => { + return await withLock(identity.environmentId, async () => { + const placement = placementBinding(identity); + const processTurn = processTurnBinding(identity); + const observed = processTurn ? observedAckCursorFor(processTurn) : undefined; + const wasNewSequence = request.seq > (observed?.liveSeq ?? 0); + const result = applyLiveEvent(identity, request); + if (!result.ok || !placement || !processTurn) { + return result; + } + const pending = pendingTerminalTurnFences.get(placement.sessionId); + if (pending && !matchesTurnBinding(pending, processTurn)) { + pendingTerminalTurnFences.delete(placement.sessionId); + } + if (isTerminalLiveEvent(request) && wasNewSequence) { + pendingTerminalTurnFences.set(placement.sessionId, { + ...processTurn, + terminalLiveSeq: request.seq, + }); + } + const terminal = pendingTerminalTurnFences.get(placement.sessionId); + if ( + terminal && + matchesTurnBinding(terminal, processTurn) && + result.result.ackedSeq >= terminal.terminalLiveSeq + ) { + // A gap fill can ACK a previously buffered terminal event. Fence from + // the observed high-water marks, not only from the request carrying it. + terminalTurnFences.set( + placement.sessionId, + observedAckCursorFor(processTurn) ?? + recordAckCursor(processTurn, { liveSeq: result.result.ackedSeq }), + ); + pendingTerminalTurnFences.delete(placement.sessionId); + } + return result; + }); + }; + + const revalidateInference = ( + identity: WorkerConnectionIdentity, + request: WorkerInferenceStartParams | WorkerInferenceCancelParams, + ): "epoch-mismatch" | "session-not-attached" | null => { + if (request.sessionId !== identity.sessionId) { + return "session-not-attached"; + } + const binding = validateAttachedWorkerRequest(identity, request.runEpoch, { + kind: "inference", + }); + return binding.ok ? null : "reason" in binding ? binding.reason : "session-not-attached"; + }; + + const startInference = ( + identity: WorkerConnectionIdentity, + request: WorkerInferenceStartParams, + sink: WorkerInferenceSink, + ): WorkerInferenceStartServiceResult => { + if (request.sessionId !== identity.sessionId || request.runId !== identity.runId) { + return { ok: false, reason: "session-not-attached" }; + } + const binding = validateAttachedWorkerRequest(identity, request.runEpoch, { + kind: "inference", + }); + if (!binding.ok) { + return binding; + } + return inference.start({ + identity, + request, + sink, + revalidate: () => revalidateInference(identity, request), + }); + }; + + const cancelInference = ( + identity: WorkerConnectionIdentity, + request: WorkerInferenceCancelParams, + ): WorkerInferenceCancelServiceResult => { + if (request.sessionId !== identity.sessionId || request.runId !== identity.runId) { + return { ok: false, reason: "session-not-attached" }; + } + const binding = validateAttachedWorkerRequest(identity, request.runEpoch, { + kind: "inference", + }); + if (!binding.ok) { + return binding; + } + return inference.cancel({ + identity, + request, + revalidate: () => revalidateInference(identity, request), + }); + }; + + return { + admitWorker: async (admission: WorkerConnectParams["admission"]) => { + if (options.isStopping()) { + return { ok: false, reason: "environment-unavailable" } as const; + } + const preflightAtMs = now(); + const preflight = admitWorkerConnection({ + store, + admission, + expectedBuild: admission.handshake, + nowMs: preflightAtMs, + allowExpiredCredential: true, + }); + if (!preflight.ok) { + return preflight; + } + if (preflightAtMs >= preflight.identity.credentialExpiresAtMs) { + const placement = placementBinding(preflight.identity); + if (!placement || !options.placementStore?.validateWorkerTurn(placement)) { + return { ok: false, reason: "credential-expired" } as const; + } + } + let expectedBuild: ExpectedWorkerBuild; + try { + expectedBuild = await options.prepareInstallation("bundle"); + } catch { + return { ok: false, reason: "environment-unavailable" } as const; + } + if (options.isStopping()) { + return { ok: false, reason: "environment-unavailable" } as const; + } + const admittedAtMs = now(); + const admitted = admitWorkerConnection({ + store, + admission, + expectedBuild, + nowMs: admittedAtMs, + allowExpiredCredential: true, + }); + if (!admitted.ok) { + return admitted; + } + const expired = admittedAtMs >= admitted.identity.credentialExpiresAtMs; + if ( + !options.placementStore || + (admitted.identity.sessionId === null && admitted.identity.runId === null) + ) { + return expired ? ({ ok: false, reason: "credential-expired" } as const) : admitted; + } + const placement = placementBinding(admitted.identity); + if (!placement || !options.placementStore.validateWorkerTurn(placement)) { + return { + ok: false, + reason: expired ? "credential-expired" : "placement-mismatch", + } as const; + } + return admitted; + }, + validateWorkerConnection: (identity: WorkerConnectionIdentity) => { + if (options.isStopping()) { + return "environment-unavailable" as const; + } + const placement = validateWorkerPlacement(identity); + if (!placement.valid) { + return "placement-mismatch" as const; + } + const environmentFailure = validateWorkerConnectionIdentity({ + store, + identity, + nowMs: now(), + }); + if ( + environmentFailure && + !(environmentFailure === "credential-expired" && placement.durableClaim) + ) { + return environmentFailure; + } + return null; + }, + commitTranscript, + pushLiveEvent, + executeSessionTool, + startInference, + cancelInference, + cancelInferenceForSession: (params: { sessionId: string; runId?: string }): string[] => + inference.cancelSession(params.sessionId, params.runId), + hasInferenceForSession: (sessionId: string, runId?: string): boolean => + inference.hasSession(sessionId, runId), + resolveInferenceSessionForRunId: (runId: string): string | undefined => + inference.resolveSessionIdForRunId(runId), + clear: () => { + observedAckCursors.clear(); + pendingTerminalTurnFences.clear(); + terminalTurnFences.clear(); + }, + }; +}