diff --git a/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift b/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift index 689ffa3bbe53..ba2a90a9cf85 100644 --- a/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift +++ b/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift @@ -20861,6 +20861,7 @@ public struct PortalSummary: Codable, Sendable { public let publicurl: String public let path: String? public let description: String? + public let origin: String? public let createdatms: Int public init( @@ -20873,6 +20874,7 @@ public struct PortalSummary: Codable, Sendable { publicurl: String, path: String? = nil, description: String? = nil, + origin: String? = nil, createdatms: Int) { self.id = id @@ -20884,6 +20886,7 @@ public struct PortalSummary: Codable, Sendable { self.publicurl = publicurl self.path = path self.description = description + self.origin = origin self.createdatms = createdatms } @@ -20897,6 +20900,7 @@ public struct PortalSummary: Codable, Sendable { case publicurl = "publicUrl" case path case description + case origin case createdatms = "createdAtMs" } } @@ -20942,6 +20946,7 @@ public struct PortalOpenResult: Codable, Sendable { public let publicurl: String public let path: String? public let description: String? + public let origin: String? public let createdatms: Int public init( @@ -20954,6 +20959,7 @@ public struct PortalOpenResult: Codable, Sendable { publicurl: String, path: String? = nil, description: String? = nil, + origin: String? = nil, createdatms: Int) { self.id = id @@ -20965,6 +20971,7 @@ public struct PortalOpenResult: Codable, Sendable { self.publicurl = publicurl self.path = path self.description = description + self.origin = origin self.createdatms = createdatms } @@ -20978,6 +20985,7 @@ public struct PortalOpenResult: Codable, Sendable { case publicurl = "publicUrl" case path case description + case origin case createdatms = "createdAtMs" } } diff --git a/config/assertion-safety-baseline.txt b/config/assertion-safety-baseline.txt index 1047a03f43b9..376d57ac684c 100644 --- a/config/assertion-safety-baseline.txt +++ b/config/assertion-safety-baseline.txt @@ -3106,7 +3106,6 @@ src/gateway/worker-environments/transcript-commit.ts 6 src/gateway/worker-environments/worker-session-tool-executor.ts 1 src/gateway/worker-environments/worker-turn-admission.ts 3 src/gateway/worker-environments/worker-turn-payload.ts 4 -src/gateway/worker-environments/worker-turn-rpc.ts 2 src/gateway/worker-environments/workspace-accepted-publication.ts 1 src/gateway/worker-environments/workspace-manifest.ts 14 src/gateway/worker-environments/workspace-path-exclusions.ts 2 @@ -3420,7 +3419,6 @@ src/model-catalog/bundled-catalog-stamp.ts 1 src/model-catalog/remote-overlay.ts 1 src/model-picker/apply-session-model-selection.ts 1 src/node-host/config.ts 1 -src/node-host/desktop-stream-command.ts 2 src/node-host/invoke-agent-cli-claude-params.ts 4 src/node-host/invoke-agent-cli-claude.ts 1 src/node-host/invoke-file-commands.ts 1 diff --git a/docs/gateway/cloud-workers.md b/docs/gateway/cloud-workers.md index ae004e71751a..0e466d0063b2 100644 --- a/docs/gateway/cloud-workers.md +++ b/docs/gateway/cloud-workers.md @@ -31,6 +31,8 @@ The bundled Crabbox cloud provider advertises both `worker-turn` and `remote-exe After Crabbox setup, the cloud node dials the Gateway's public TLS endpoint over outbound WebSocket. Worker control, Codex remote execution, and workspace transfer use authenticated node or worker channels, not a Gateway-created reverse tunnel or rsync. Crabbox itself may still require SSH reachability while its CLI runs the provider-owned setup command. Outbound internet access is provider policy; the default AWS profile can reach the internet unless you restrict its network or security group. +OpenClaw `worker-turn` sessions can open [portals](/gateway/portals) on node-backed cloud workers, including the bundled Crabbox provider. For each proxied HTTP or WebSocket connection, the enrolled node redeems a single-use ticket over a TLS-pinned WebSocket to the Gateway and connects to the worker's selected loopback port. This preserves the existing **Control UI → Portals** experience, authentication, and live reload without opening inbound worker ports or creating an SSH tunnel. The tool is available only when the node advertises portal-stream support; older node bundles do not receive it. SSH-backed `remote-exec` placements, including Codex sessions, do not run the OpenClaw worker tool loop, so the `portal` tool does not apply there. Update an unsupported node or move the session back to the Gateway with `sessions.move` when a Gateway-hosted portal is needed. + For a loopback Gateway behind public HTTPS ingress, set `gateway.publicOrigin` to the proxy's bare origin. Node enrollment uses it as the default external pairing endpoint; `plugins.entries.device-pair.config.publicUrl` remains the pairing-specific override. If either URL is behind a reverse proxy, including cloudflared, nginx, or externally managed Tailscale Serve, `gateway.trustedProxies` must include the proxy's source address (typically loopback for a same-host proxy). Otherwise, forwarded client headers cause node enrollment to fail with `proxy_attribution_required`. ## Requirements @@ -402,6 +404,7 @@ The desktop never gains public ingress. The node reads `/var/lib/crabbox/vnc.pas - **Cloud destinations are hidden or an RPC is denied** — cloud profile dispatch and profile-target moves require `operator.admin`. `operator.write` can dispatch or move to an eligible paired device, move to the Gateway, and reclaim a placement; `operator.read` alone can discover profiles but cannot start, stop, or move a session. Profile configuration, infrastructure pairing, Connect machine, raw environment lifecycle, direct `execNode` execution, incognito sessions, and arbitrary host or node paths remain `operator.admin`. - **The selected runtime lacks cloud placement support** — choose a model whose advertised runtime supports cloud placement. The bundled OpenClaw and Codex runtimes are supported; undeclared runtimes remain local-only. - **Codex cannot use a cloud profile** — verify that the profile advertises `remote-exec`, its setup installs the exact official `npm:@openclaw/codex@` plugin, the Gateway enables the Codex plugin, and `gateway.nodes.commands.allow` includes `codex.exec-server.stdio.v1` without a matching deny rule. Approve the exact node invocation when prompted. Codex does not require an available OpenClaw worker slot; a missing plugin or denied command must be corrected rather than bypassed with Gateway or SSH execution. +- **The portal tool is unavailable on a worker** — confirm the session uses OpenClaw `worker-turn` on an enrolled node that advertises portal-stream support. Update older node bundles when necessary. SSH-backed `remote-exec` placements, including Codex sessions, do not run the OpenClaw worker tool loop; move the session back to the Gateway with `sessions.move` when a Gateway-hosted portal is needed. - **"Worker bootstrap requires Node.js on the leased host"** — add a Node install to `settings.setup` (see above). - **AWS instance-role attestation fails** — clear `aws.instanceProfile` (and `CRABBOX_AWS_INSTANCE_PROFILE`, if set). Install Crabbox 0.41.1 or newer; older binaries do not satisfy the fixed-ID and authoritative `providerMetadata.instanceProfileAttached` contracts required for AWS admission. - **Dispatch or workspace recovery fails** — inspect `environments.list` and `sessions.describe`. A failed environment exposes its bounded environment error. A failed placement exposes `recoveryError` plus its durable per-session `terminalReason`; the selected Control UI chat shows that terminal reason above the composer. When deeper diagnosis is necessary, an operator on the Gateway host can inspect the durable worker state read-only. Do not edit the state database to bypass lifecycle fencing. diff --git a/docs/gateway/portals.md b/docs/gateway/portals.md index 6f0c128a4a5c..964d17fe74a1 100644 --- a/docs/gateway/portals.md +++ b/docs/gateway/portals.md @@ -7,7 +7,7 @@ read_when: - Troubleshooting portal access or live reload --- -Portals expose a development server running on the Gateway host to the operator's browser. They proxy HTTP and WebSockets for live reload and appear in **Control UI → Portals**. +Portals expose a development server running on the Gateway host or a node-backed cloud worker to the operator's browser. They proxy HTTP and WebSockets for live reload and appear in **Control UI → Portals**. ## Quick start @@ -18,6 +18,8 @@ Ask the agent to open a portal: The agent opens a portal for the application's port, then starts the development server with a background `exec` call. Opening a portal only creates the proxy listener; it does not inject environment variables into your server. The agent sets `PORT` (the port it opened) and `PUBLIC_URL` (the portal's public base URL) in that `exec` command's own environment, so the app binds the expected port and generates correct absolute URLs. +For a session on a node-backed cloud worker, including the bundled Crabbox provider, the development server runs on the worker. Each portal connection receives its own single-use ticket, which the enrolled node redeems over a TLS-pinned WebSocket to the Gateway before connecting to the selected loopback port. This uses the existing authenticated node channel without exposing the worker to inbound traffic or creating an SSH tunnel. Stopping or replacing the worker closes its portals. + ## Declare development servers Optionally commit `.openclaw/portals.json` to the workspace repository so the agent can discover the available development servers: @@ -64,6 +66,7 @@ Out of the box: - `portal` belongs to `group:ui` and the `coding` profile, so coding agents have it while `messaging` and `minimal` agents do not. - Sandboxed sessions never receive it, because opening a portal starts a listener on the Gateway host. - It is blocked for HTTP `POST /tools/invoke` and restricted to the session owner, the same treatment `terminal` gets. +- Cloud-worker sessions receive it only when their enrolled node advertises portal-stream support. Older node bundles without that capability do not receive the tool. To turn portals off everywhere, deny the tool in the global policy: @@ -91,11 +94,12 @@ Each portal uses a separate origin on its own port and binds to the same interfa Browser cookies are hostname-scoped rather than port-scoped, so the proxy gives each portal instance a random `oc_portal__` cookie-name prefix. Requests forward only cookies with the current portal's prefix and strip it before reaching the application; Gateway cookies, unprefixed cookies, and cookies from sibling or closed portals are dropped. Application `Set-Cookie` responses receive the prefix, and any `Domain` attribute is removed so the cookie stays host-only. -Portals proxy only the selected local development server. They never serve Gateway data, and every portal ends when the Gateway restarts. +Portals proxy only the selected development server on the Gateway host or a node-backed cloud worker. Worker connections use single-use tickets and the enrolled node's TLS-pinned Gateway connection; they never expose a public worker port or require SSH forwarding. Portals never serve Gateway data, and every portal ends when the Gateway restarts. ## Limitations -- The development server must run on the Gateway host. Remote worker support is planned. +- Older node bundles without portal-stream support cannot open worker portals. Update the node bundle, or move the session back to the Gateway with `sessions.move`. +- SSH-backed `remote-exec` placements, including Codex sessions, do not run the OpenClaw worker tool loop, so the `portal` tool does not apply there. Move the session back to the Gateway with `sessions.move` when a Gateway-hosted portal is needed. - A proxy or tunnel in front of the Gateway does not automatically expose portal listener ports. The Control UI detects this and shows a reachable URL with retry guidance instead of mounting a dead iframe. - The prefix isolates cookies forwarded to each target; it does not create separate browser cookie jars. Browser-side code can see non-`HttpOnly` cookies for sibling portals on the same hostname through `document.cookie`. Use `HttpOnly` for sensitive application cookies. Applications that manage cookies in browser code must account for the prefix; unprefixed cookies written directly by browser code are not forwarded to the target. @@ -103,7 +107,7 @@ Portals proxy only the selected local development server. They never serve Gatew ### The portal shows a 502 waiting page -The proxy is ready, but the application is not listening on the selected port. The page retries automatically. Check the background process and confirm that the server honors `PORT`. +The proxy is ready, but the application is not listening on the selected port or its worker node is temporarily disconnected. The page retries automatically. Check the background process, confirm that the server honors `PORT`, and verify that the worker node is connected. ### The portal is not reachable from this browser diff --git a/packages/gateway-protocol/src/public-schema.ts b/packages/gateway-protocol/src/public-schema.ts index c7300e6917cc..f2b01ce9ac5c 100644 --- a/packages/gateway-protocol/src/public-schema.ts +++ b/packages/gateway-protocol/src/public-schema.ts @@ -37,6 +37,8 @@ export { WorkerSessionsSpawnResponseFrameSchema, WorkerSessionsSendParamsSchema, WorkerSessionsSendResponseFrameSchema, + WorkerPortalParamsSchema, + WorkerPortalResponseFrameSchema, WorkerSessionToolResultSchema, WorkerLiveEventSchema, WorkerLiveEventErrorDetailsSchema, @@ -58,6 +60,7 @@ export { WORKER_LAUNCH_V2_PROTOCOL_FEATURE, WORKER_LIVE_EVENT_PROTOCOL_FEATURE, WORKER_SESSION_TOOLS_PROTOCOL_FEATURE, + WORKER_PORTAL_PROTOCOL_FEATURE, WORKER_SESSION_TOOL_MAX_TEXT_LENGTH, WORKER_PROTOCOL_FEATURES, WORKER_PROTOCOL_MAX_FEATURE_LENGTH, diff --git a/packages/gateway-protocol/src/schema/portals.ts b/packages/gateway-protocol/src/schema/portals.ts index 6eee3cc4dd60..e5e46ff83d3b 100644 --- a/packages/gateway-protocol/src/schema/portals.ts +++ b/packages/gateway-protocol/src/schema/portals.ts @@ -13,6 +13,7 @@ const PortalSummaryMetadataFields = { publicUrl: NonEmptyString, path: Type.Optional(Type.String({ pattern: "^/" })), description: Type.Optional(Type.String()), + origin: Type.Optional(Type.String()), createdAtMs: Type.Integer({ minimum: 0 }), }; diff --git a/packages/gateway-protocol/src/schema/worker-admission.test.ts b/packages/gateway-protocol/src/schema/worker-admission.test.ts index b2998949a2e7..bf7c98ce10b7 100644 --- a/packages/gateway-protocol/src/schema/worker-admission.test.ts +++ b/packages/gateway-protocol/src/schema/worker-admission.test.ts @@ -11,6 +11,7 @@ import { WorkerLiveEventRequestFrameSchema, WorkerLiveEventResponseFrameSchema, WorkerProtocolCloseReasonSchema, + WorkerPortalResponseFrameSchema, WorkerSessionsSendResponseFrameSchema, WorkerSessionsSpawnResponseFrameSchema, WorkerTranscriptCommitRequestFrameSchema, @@ -19,6 +20,7 @@ import { WORKER_LAUNCH_V2_PROTOCOL_FEATURE, WORKER_GITHUB_PUBLICATION_PROTOCOL_FEATURE, WORKER_PROTOCOL_FEATURES, + WORKER_PORTAL_PROTOCOL_FEATURE, WORKER_PROTOCOL_MAX_FRAME_ID_LENGTH, WORKER_PROTOCOL_MAX_PAYLOAD_BYTES, WORKER_RPC_SET_VERSION, @@ -30,6 +32,7 @@ import { validateWorkerHeartbeatParams, validateWorkerGitHubPublishParams, validateWorkerLiveEventParams, + validateWorkerPortalParams, validateWorkerSessionsSendParams, validateWorkerSessionsSpawnParams, validateWorkerTranscriptCommitParams, @@ -284,9 +287,11 @@ describe("worker protocol schemas", () => { message: "report status", }; const publish = { toolCallId: "call-publish", title: "Publish the fix" }; + const portal = { toolCallId: "call-portal", action: "open", port: 3000, path: "/app" }; expect(validateWorkerSessionsSpawnParams(spawn)).toBe(true); expect(validateWorkerSessionsSendParams(send)).toBe(true); expect(validateWorkerGitHubPublishParams(publish)).toBe(true); + expect(validateWorkerPortalParams(portal)).toBe(true); expect(validateWorkerSessionsSpawnParams({ ...spawn, unexpected: true })).toBe(false); expect(validateWorkerSessionsSendParams({ ...send, message: "" })).toBe(false); expect(validateWorkerGitHubPublishParams({ ...publish, token: "secret" })).toBe(false); @@ -296,6 +301,10 @@ describe("worker protocol schemas", () => { expect(validateWorkerGitHubPublishParams({ ...publish, branch: "main" })).toBe(false); expect(validateWorkerGitHubPublishParams({ ...publish, title: "Fix\rInject" })).toBe(false); expect(validateWorkerGitHubPublishParams({ ...publish, commitMessage: "Inject" })).toBe(false); + expect(validateWorkerPortalParams({ ...portal, token: "secret" })).toBe(false); + expect(validateWorkerPortalParams({ ...portal, action: "unknown" })).toBe(false); + expect(validateWorkerPortalParams({ ...portal, port: 0 })).toBe(false); + expect(validateWorkerPortalParams({ ...portal, path: "app" })).toBe(false); const escaped = "\0"; const requestBytes = (method: string, requestParams: object) => Buffer.byteLength( @@ -357,6 +366,22 @@ describe("worker protocol schemas", () => { expect(requestBytes("worker.github.publish", maximalPublish)).toBeLessThanOrEqual( WORKER_PROTOCOL_MAX_PAYLOAD_BYTES, ); + const maximalPortal = { + toolCallId: escaped.repeat(256), + action: "open", + port: 65_535, + title: escaped.repeat(256), + description: escaped.repeat(WORKER_SESSION_TOOL_MAX_TEXT_LENGTH), + path: `/${escaped.repeat(1_023)}`, + id: escaped.repeat(256), + }; + expect(validateWorkerPortalParams(maximalPortal)).toBe(true); + expect(requestBytes("worker.portal", maximalPortal)).toBeLessThanOrEqual( + WORKER_PROTOCOL_MAX_PAYLOAD_BYTES, + ); + expect(validateWorkerPortalParams({ ...maximalPortal, description: impossibleText })).toBe( + false, + ); expect( validateWorkerSessionsSpawnParams({ ...spawn, @@ -365,6 +390,7 @@ describe("worker protocol schemas", () => { ).toBe(false); expect(WORKER_PROTOCOL_FEATURES).toContain(WORKER_SESSION_TOOLS_PROTOCOL_FEATURE); expect(WORKER_PROTOCOL_FEATURES).toContain(WORKER_GITHUB_PUBLICATION_PROTOCOL_FEATURE); + expect(WORKER_PROTOCOL_FEATURES).toContain(WORKER_PORTAL_PROTOCOL_FEATURE); const response = { type: "res" as const, @@ -375,6 +401,7 @@ describe("worker protocol schemas", () => { expect(Value.Check(WorkerSessionsSpawnResponseFrameSchema, response)).toBe(true); expect(Value.Check(WorkerSessionsSendResponseFrameSchema, response)).toBe(true); expect(Value.Check(WorkerGitHubPublishResponseFrameSchema, response)).toBe(true); + expect(Value.Check(WorkerPortalResponseFrameSchema, response)).toBe(true); expect( Value.Check(WorkerSessionsSendResponseFrameSchema, { ...response, diff --git a/packages/gateway-protocol/src/schema/worker-admission.ts b/packages/gateway-protocol/src/schema/worker-admission.ts index 709e08af115c..3c342a2fbc88 100644 --- a/packages/gateway-protocol/src/schema/worker-admission.ts +++ b/packages/gateway-protocol/src/schema/worker-admission.ts @@ -42,6 +42,7 @@ export const WORKER_PROTOCOL_METHODS = [ "worker.sessions.spawn", "worker.sessions.send", "worker.github.publish", + "worker.portal", ] as const; export const WORKER_TRANSCRIPT_COMMIT_PROTOCOL_FEATURE = "worker-transcript-commit-v1"; export const WORKER_LIVE_EVENT_PROTOCOL_FEATURE = "worker-live-event-v1"; @@ -49,6 +50,7 @@ export const WORKER_LAUNCH_V2_PROTOCOL_FEATURE = "worker-launch-v2"; export const WORKER_EXECUTION_CONTEXT_PROTOCOL_FEATURE = "worker-execution-context-v2"; export const WORKER_SESSION_TOOLS_PROTOCOL_FEATURE = "worker-session-tools-v1"; export const WORKER_GITHUB_PUBLICATION_PROTOCOL_FEATURE = "worker-github-publication-v1"; +export const WORKER_PORTAL_PROTOCOL_FEATURE = "worker-portal-v1"; export const WORKER_PROTOCOL_FEATURES = [ "worker-heartbeat-v1", WORKER_TRANSCRIPT_COMMIT_PROTOCOL_FEATURE, @@ -58,6 +60,7 @@ export const WORKER_PROTOCOL_FEATURES = [ WORKER_EXECUTION_CONTEXT_PROTOCOL_FEATURE, WORKER_SESSION_TOOLS_PROTOCOL_FEATURE, WORKER_GITHUB_PUBLICATION_PROTOCOL_FEATURE, + WORKER_PORTAL_PROTOCOL_FEATURE, "worker-inference-v1", ] as const; export const WORKER_PROTOCOL_MAX_METHOD_LENGTH = 64; @@ -230,11 +233,21 @@ export const WorkerGitHubPublishParamsSchema = closedObject({ body: Type.Optional(GitHubPublicationBodySchema), }); +export const WorkerPortalParamsSchema = closedObject({ + toolCallId: WorkerSessionToolCallIdSchema, + action: Type.Union([Type.Literal("open"), Type.Literal("list"), Type.Literal("close")]), + port: Type.Optional(Type.Integer({ minimum: 1, maximum: 65_535 })), + title: Type.Optional(Type.String({ minLength: 1, maxLength: 256 })), + description: Type.Optional(Type.String({ maxLength: WORKER_SESSION_TOOL_MAX_TEXT_LENGTH })), + path: Type.Optional(Type.String({ maxLength: 1_024, pattern: "^/" })), + id: Type.Optional(Type.String({ minLength: 1, maxLength: 256 })), +}); + export const WorkerSessionToolResultSchema = closedObject({ resultJson: Type.String({ minLength: 2, maxLength: WORKER_PROTOCOL_MAX_PAYLOAD_BYTES }), }); -export const WorkerSessionsSpawnResponseFrameSchema = Type.Union([ +const WorkerSessionToolResponseFrameSchema = Type.Union([ closedObject({ type: Type.Literal("res"), id: WorkerFrameIdSchema, @@ -244,25 +257,10 @@ export const WorkerSessionsSpawnResponseFrameSchema = Type.Union([ WorkerErrorResponseFrameSchema, ]); -export const WorkerSessionsSendResponseFrameSchema = Type.Union([ - closedObject({ - type: Type.Literal("res"), - id: WorkerFrameIdSchema, - ok: Type.Literal(true), - payload: WorkerSessionToolResultSchema, - }), - WorkerErrorResponseFrameSchema, -]); - -export const WorkerGitHubPublishResponseFrameSchema = Type.Union([ - closedObject({ - type: Type.Literal("res"), - id: WorkerFrameIdSchema, - ok: Type.Literal(true), - payload: WorkerSessionToolResultSchema, - }), - WorkerErrorResponseFrameSchema, -]); +export const WorkerSessionsSpawnResponseFrameSchema = WorkerSessionToolResponseFrameSchema; +export const WorkerSessionsSendResponseFrameSchema = WorkerSessionToolResponseFrameSchema; +export const WorkerGitHubPublishResponseFrameSchema = WorkerSessionToolResponseFrameSchema; +export const WorkerPortalResponseFrameSchema = WorkerSessionToolResponseFrameSchema; const WorkerTranscriptTextContentSchema = closedObject({ type: Type.Literal("text"), @@ -728,6 +726,7 @@ export type WorkerHeartbeatResponseFrame = Static; export type WorkerSessionsSendParams = Static; export type WorkerGitHubPublishParams = Static; +export type WorkerPortalParams = Static; export type WorkerSessionToolResult = Static; export type WorkerSessionsSpawnResponseFrame = Static< typeof WorkerSessionsSpawnResponseFrameSchema @@ -736,6 +735,7 @@ export type WorkerSessionsSendResponseFrame = Static; +export type WorkerPortalResponseFrame = Static; export type WorkerTranscriptMessage = Static; export type WorkerProviderReplayState = Static; export type WorkerTranscriptCommitParams = Static; diff --git a/packages/gateway-protocol/src/server-capabilities.ts b/packages/gateway-protocol/src/server-capabilities.ts index 2a963d922d28..ff4406e1419d 100644 --- a/packages/gateway-protocol/src/server-capabilities.ts +++ b/packages/gateway-protocol/src/server-capabilities.ts @@ -5,6 +5,7 @@ export const GATEWAY_SERVER_CAPS = { GATEWAY_RESTART_TARGET_SAFE: "gateway-restart-target-safe-v1", NODE_WORKER_BUNDLE_RETENTION: "node-worker-bundle-retention-v1", NODE_WORKER_BUNDLE_STATUS: "node-worker-bundle-status-v1", + NODE_WORKER_PORTAL_STREAM: "node-worker-portal-stream-v1", SESSION_UNREAD_ACK_CONTRACT: "session-unread-ack-contract", SYSTEM_AGENT_WIZARD_CANCEL: "openclaw-chat-wizard-cancel", SYSTEM_AGENT_SETUP_MODEL_REF: "openclaw-setup-model-ref", diff --git a/packages/gateway-protocol/src/validator-registry.ts b/packages/gateway-protocol/src/validator-registry.ts index 8e97f08c9cd2..572ac7f522eb 100644 --- a/packages/gateway-protocol/src/validator-registry.ts +++ b/packages/gateway-protocol/src/validator-registry.ts @@ -22,6 +22,7 @@ export const validateWorkerConnectRequestFrame = compile(S.WorkerConnectRequestF export const validateWorkerHeartbeatParams = compile(S.WorkerHeartbeatParamsSchema); export const validateWorkerSessionsSpawnParams = compile(S.WorkerSessionsSpawnParamsSchema); export const validateWorkerSessionsSendParams = compile(S.WorkerSessionsSendParamsSchema); +export const validateWorkerPortalParams = compile(S.WorkerPortalParamsSchema); function checkWorkerProtocolJson(data: unknown): ValidationError | undefined { const stack: Array<{ depth: number; value: unknown }> = [{ depth: 0, value: data }]; diff --git a/src/agents/tools/portal-tool-contract.ts b/src/agents/tools/portal-tool-contract.ts new file mode 100644 index 000000000000..e5d9805eac86 --- /dev/null +++ b/src/agents/tools/portal-tool-contract.ts @@ -0,0 +1,27 @@ +import { Type } from "typebox"; +import { + PortalCloseResultSchema, + PortalListResultSchema, + PortalSummarySchema, +} from "../../../packages/gateway-protocol/src/schema/portals.js"; + +export const PORTAL_TOOL_DESCRIPTION = + "Expose local HTTP server; operator sees it live in Control UI. Order matters: action=open with the port first, which returns the URL; then start the dev server as a background process, passing PORT and PUBLIC_URL from that result. Workspace may declare servers in .openclaw/portals.json. Proxies HTTP and WebSockets, so hot reload works; serves retry page until port listens. action=list and action=close manage portals. Portals end at gateway restart."; + +export const PortalToolSchema = Type.Object( + { + action: Type.String({ enum: ["open", "list", "close"], description: "Portal action" }), + port: Type.Optional(Type.Integer({ minimum: 1, maximum: 65_535 })), + title: Type.Optional(Type.String({ minLength: 1 })), + description: Type.Optional(Type.String()), + path: Type.Optional(Type.String({ pattern: "^/" })), + id: Type.Optional(Type.String({ minLength: 1 })), + }, + { additionalProperties: false }, +); + +export const PortalOutputSchema = Type.Union([ + PortalSummarySchema, + PortalListResultSchema, + PortalCloseResultSchema, +]); diff --git a/src/agents/tools/portal-tool.ts b/src/agents/tools/portal-tool.ts index c212d5e95b6e..67ad231da851 100644 --- a/src/agents/tools/portal-tool.ts +++ b/src/agents/tools/portal-tool.ts @@ -1,11 +1,7 @@ -import { Type } from "typebox"; -import { - PortalCloseResultSchema, - PortalListResultSchema, - PortalSummarySchema, - type PortalCloseResult, - type PortalListResult, - type PortalSummary, +import type { + PortalCloseResult, + PortalListResult, + PortalSummary, } from "../../../packages/gateway-protocol/src/index.js"; import { WRITE_SCOPE } from "../../gateway/operator-scopes.js"; import type { AgentToolResult } from "../runtime/index.js"; @@ -22,37 +18,36 @@ import { type AgentToolGatewayRequestCaller, type InProcessGatewayCaller, } from "./in-process-gateway.js"; +import { + PORTAL_TOOL_DESCRIPTION, + PortalOutputSchema, + PortalToolSchema, +} from "./portal-tool-contract.js"; -const PORTAL_ACTIONS = ["open", "list", "close"] as const; // Reading a portal's bearer URL is a write-scope capability: it is the same // credential action=open mints, so listing must ask for it explicitly. const PORTAL_URL_SCOPE = WRITE_SCOPE; -const PortalToolSchema = Type.Object( - { - action: Type.String({ enum: [...PORTAL_ACTIONS], description: "Portal action" }), - port: Type.Optional(Type.Integer({ minimum: 1, maximum: 65_535 })), - title: Type.Optional(Type.String({ minLength: 1 })), - description: Type.Optional(Type.String()), - path: Type.Optional(Type.String({ pattern: "^/" })), - id: Type.Optional(Type.String({ minLength: 1 })), - }, - { additionalProperties: false }, -); - -const PortalToolOutputSchema = Type.Union([ - PortalSummarySchema, - PortalListResultSchema, - PortalCloseResultSchema, -]); - type PortalToolOptions = { callGateway?: InProcessGatewayCaller; callGatewayRequest?: AgentToolGatewayRequestCaller; }; -function portalResult(text: string, payload: T): AgentToolResult { - const result = jsonResult(payload); +type PortalToolOutcome = + | { action: "open"; result: PortalSummary } + | { action: "list"; result: PortalListResult } + | { action: "close"; id: string; result: PortalCloseResult }; + +export function formatPortalResult( + outcome: PortalToolOutcome, +): AgentToolResult { + const text = + outcome.action === "open" + ? `Portal available at ${outcome.result.url}. Pass PUBLIC_URL=${outcome.result.publicUrl} and PORT=${outcome.result.port} when starting the dev server. The operator can see it in the Control UI Portals page.` + : outcome.action === "list" + ? `${outcome.result.portals.length} active portal${outcome.result.portals.length === 1 ? "" : "s"}. The operator can see them in the Control UI Portals page.` + : `Portal ${outcome.id} closed. The Control UI Portals page has been updated.`; + const result = jsonResult(outcome.result); return { ...result, content: [{ type: "text", text }, ...result.content] }; } @@ -62,10 +57,9 @@ export function createPortalTool(options: PortalToolOptions = {}): AnyAgentTool return { label: "Portal", name: "portal", - description: - "Expose local HTTP server; operator sees it live in Control UI. Order matters: action=open with the port first, which returns the URL; then start the dev server as a background process, passing PORT and PUBLIC_URL from that result. Workspace may declare servers in .openclaw/portals.json. Proxies HTTP and WebSockets, so hot reload works; serves retry page until port listens. action=list and action=close manage portals. Portals end at gateway restart.", + description: PORTAL_TOOL_DESCRIPTION, parameters: PortalToolSchema, - outputSchema: PortalToolOutputSchema, + outputSchema: PortalOutputSchema, execute: async (_toolCallId, rawArgs) => { const params = rawArgs as Record; const action = readToolStringParam(params, "action", { required: true }); @@ -79,18 +73,12 @@ export function createPortalTool(options: PortalToolOptions = {}): AnyAgentTool params: {}, scopes: [PORTAL_URL_SCOPE], }); - return portalResult( - `${result.portals.length} active portal${result.portals.length === 1 ? "" : "s"}. The operator can see them in the Control UI Portals page.`, - result, - ); + return formatPortalResult({ action: "list", result }); } if (action === "close") { const id = readToolStringParam(params, "id", { required: true }); const result = await callGateway("portal.close", { id }); - return portalResult( - `Portal ${id} closed. The Control UI Portals page has been updated.`, - result, - ); + return formatPortalResult({ action: "close", id, result }); } if (action !== "open") { throw new ToolInputError(`Unknown portal action: ${action}`); @@ -114,10 +102,7 @@ export function createPortalTool(options: PortalToolOptions = {}): AnyAgentTool ...(description !== undefined ? { description } : {}), ...(path !== undefined ? { path } : {}), }); - return portalResult( - `Portal available at ${portal.url}. Pass PUBLIC_URL=${portal.publicUrl} and PORT=${portal.port} when starting the dev server. The operator can see it in the Control UI Portals page.`, - portal, - ); + return formatPortalResult({ action: "open", result: portal }); }, }; } diff --git a/src/gateway/desktop/node-stream-broker.test.ts b/src/gateway/desktop/node-stream-broker.test.ts index 5df71eb38ecb..cf4a2be08a1d 100644 --- a/src/gateway/desktop/node-stream-broker.test.ts +++ b/src/gateway/desktop/node-stream-broker.test.ts @@ -271,3 +271,54 @@ describe("node desktop stream tickets", () => { releaseCheck(); }); }); + +describe("node portal stream tickets", () => { + it("pairs a ready portal stream and keeps desktop and portal tickets isolated", async () => { + const broker = createNodeDesktopStreamBroker(); + const session = { connId: "conn-1", pairingGeneration: "generation-1" }; + const baseUrl = await startBrokerServer({ broker, session }); + const minted = broker.mintPortal({ nodeId: "node-1", ...session }); + + await expectUnauthorized( + `${baseUrl}${minted.attachPath.replace("/node-portal/", "/node-desktop/")}`, + ); + const ws = await connectAndSend(`${baseUrl}${minted.attachPath}`, { ok: true }); + const attached = await minted.attached; + const response = new Promise((resolve) => { + attached.stream.once("data", resolve); + }); + ws.send(Buffer.from("HTTP/1.1 200 OK\r\n\r\n"), { binary: true }); + + await expect(response).resolves.toEqual(Buffer.from("HTTP/1.1 200 OK\r\n\r\n")); + attached.stream.destroy(); + await expectUnauthorized(`${baseUrl}${minted.attachPath}`); + }); + + it("rejects unexpected portal readiness metadata", async () => { + const broker = createNodeDesktopStreamBroker(); + const session = { connId: "conn-1", pairingGeneration: "generation-1" }; + const baseUrl = await startBrokerServer({ broker, session }); + const minted = broker.mintPortal({ nodeId: "node-1", ...session }); + + await connectAndSend(`${baseUrl}${minted.attachPath}`, { ok: true, auth: "vnc-password" }); + + await expect(minted.attached).rejects.toThrow("invalid node portal attach metadata"); + }); + + it("rejects immediately when the node closes before the target is ready", async () => { + const broker = createNodeDesktopStreamBroker(); + const session = { connId: "conn-1", pairingGeneration: "generation-1" }; + const baseUrl = await startBrokerServer({ broker, session }); + const minted = broker.mintPortal({ nodeId: "node-1", ...session }); + const ws = new WebSocket(`${baseUrl}${minted.attachPath}`); + cleanups.push(async () => ws.terminate()); + await new Promise((resolve, reject) => { + ws.once("open", resolve); + ws.once("error", reject); + }); + + ws.close(); + + await expect(minted.attached).rejects.toThrow("node portal stream closed before attach"); + }); +}); diff --git a/src/gateway/desktop/node-stream-broker.ts b/src/gateway/desktop/node-stream-broker.ts index abf57fdedef1..8e550e47ff75 100644 --- a/src/gateway/desktop/node-stream-broker.ts +++ b/src/gateway/desktop/node-stream-broker.ts @@ -4,7 +4,10 @@ import type { Duplex } from "node:stream"; import { isRecord } from "@openclaw/normalization-core/record-coerce"; import { createWebSocketStream, WebSocket, WebSocketServer, type RawData } from "ws"; import { registerSecretValueForRedaction } from "../../logging/secret-redaction-registry.js"; -import { NODE_DESKTOP_ATTACH_PATH } from "../../shared/node-desktop-stream.js"; +import { + NODE_DESKTOP_ATTACH_PATH, + NODE_PORTAL_ATTACH_PATH, +} from "../../shared/node-desktop-stream.js"; import type { NodeRegistry } from "../node-registry.js"; const DEFAULT_TICKET_TTL_MS = 60_000; @@ -17,6 +20,8 @@ type NodeDesktopStreamMetadata = { }; type AttachedNodeDesktopStream = NodeDesktopStreamMetadata & { stream: Duplex }; +type AttachedNodePortalStream = { stream: Duplex }; +type NodeStreamKind = "desktop" | "portal"; type NodeDesktopStreamBinding = { nodeId: string; @@ -25,9 +30,10 @@ type NodeDesktopStreamBinding = { }; type TicketEntry = { + kind: NodeStreamKind; binding: NodeDesktopStreamBinding; expiresAtMs: number; - resolve: (attached: AttachedNodeDesktopStream) => void; + resolve: (stream: Duplex, metadata: NodeDesktopStreamMetadata | undefined) => void; reject: (error: Error) => void; timer: ReturnType; redeemed: boolean; @@ -51,16 +57,26 @@ function rawDataBuffer(data: RawData): Buffer { return Buffer.from(data); } -function parseStreamMetadata(data: RawData, isBinary: boolean): NodeDesktopStreamMetadata { +function parseStreamMetadata( + data: RawData, + isBinary: boolean, + kind: NodeStreamKind, +): NodeDesktopStreamMetadata | undefined { const buffer = rawDataBuffer(data); if (!isBinary || buffer.length === 0 || buffer.length > MAX_ATTACH_FRAME_BYTES) { - throw new Error("invalid node desktop attach metadata"); + throw new Error(`invalid node ${kind} attach metadata`); } let value: unknown; try { value = JSON.parse(buffer.toString("utf8")); } catch { - throw new Error("invalid node desktop attach metadata"); + throw new Error(`invalid node ${kind} attach metadata`); + } + if (kind === "portal") { + if (!isRecord(value) || value.ok !== true || Object.keys(value).length !== 1) { + throw new Error("invalid node portal attach metadata"); + } + return undefined; } if (!isRecord(value) || (value.auth !== "vnc-password" && value.auth !== "ard-account")) { throw new Error("invalid node desktop attach metadata"); @@ -89,8 +105,9 @@ function writeUnauthorized(socket: Duplex): void { function readAttachedStream( ws: WebSocket, + kind: NodeStreamKind, onStreamError: (error: Error) => void, -): Promise<{ metadata: NodeDesktopStreamMetadata; stream: Duplex }> { +): Promise<{ metadata: NodeDesktopStreamMetadata | undefined; stream: Duplex }> { return new Promise((resolve, reject) => { const cleanup = () => { ws.off("message", onMessage); @@ -99,12 +116,12 @@ function readAttachedStream( const onMessage = (data: RawData, isBinary: boolean) => { cleanup(); try { - const metadata = parseStreamMetadata(data, isBinary); - // Install the stream listener in the same message turn. The pairing - // recheck may yield, but early RFB banner bytes must already be buffered. + const metadata = parseStreamMetadata(data, isBinary, kind); + // Install the stream listener before the pairing recheck yields so early + // desktop banner or portal response bytes cannot disappear. const stream = createWebSocketStream(ws, { allowHalfOpen: false }); // Retain a safety listener through the asynchronous registry handoff; - // the observer bridge adds its own lifecycle handler after claiming it. + // the consumer adds its own lifecycle handler after claiming the stream. stream.on("error", onStreamError); resolve({ metadata, @@ -116,7 +133,7 @@ function readAttachedStream( }; const onClose = () => { cleanup(); - reject(new Error("node desktop stream closed before attach")); + reject(new Error(`node ${kind} stream closed before attach`)); }; const onError = (error: Error) => { cleanup(); @@ -131,7 +148,7 @@ function readAttachedStream( }); } -/** Owns one-time node stream tickets and turns redeemed WebSockets into RFB duplexes. */ +/** Owns one-time node stream tickets and pairs authenticated desktop or portal duplexes. */ export function createNodeDesktopStreamBroker(deps: { ttlMs?: number; now?: () => number } = {}) { const ttlMs = deps.ttlMs ?? DEFAULT_TICKET_TTL_MS; const now = deps.now ?? Date.now; @@ -155,35 +172,51 @@ export function createNodeDesktopStreamBroker(deps: { ttlMs?: number; now?: () = } entry.settled = true; entry.reject(error); - entry.ws?.close(1008, "node desktop attach rejected"); + entry.ws?.close(1008, `node ${entry.kind} attach rejected`); entry.socket?.destroy(); }; - const resolveTicket = (ticket: string, attached: AttachedNodeDesktopStream): void => { + const resolveTicket = ( + ticket: string, + stream: Duplex, + metadata: NodeDesktopStreamMetadata | undefined, + ): void => { const entry = remove(ticket); if (!entry || entry.settled) { - attached.stream.destroy(); + stream.destroy(); return; } entry.settled = true; - entry.resolve(attached); + entry.resolve(stream, metadata); }; - function mint(binding: NodeDesktopStreamBinding) { + function mintStream( + binding: NodeDesktopStreamBinding, + kind: NodeStreamKind, + attach: (stream: Duplex, metadata: NodeDesktopStreamMetadata | undefined) => T, + ) { const ticket = crypto.randomBytes(24).toString("hex"); const expiresAtMs = now() + ttlMs; - let resolve!: (attached: AttachedNodeDesktopStream) => void; + let resolve!: (stream: Duplex, metadata: NodeDesktopStreamMetadata | undefined) => void; let reject!: (error: Error) => void; - const attached = new Promise((resolvePromise, rejectPromise) => { - resolve = resolvePromise; + const attached = new Promise((resolvePromise, rejectPromise) => { + resolve = (stream, metadata) => { + try { + resolvePromise(attach(stream, metadata)); + } catch (error) { + stream.destroy(); + rejectPromise(error instanceof Error ? error : new Error(String(error))); + } + }; reject = rejectPromise; }); void attached.catch(() => undefined); const timer = setTimeout(() => { - rejectTicket(ticket, new Error("node desktop stream ticket expired")); + rejectTicket(ticket, new Error(`node ${kind} stream ticket expired`)); }, ttlMs); timer.unref?.(); tickets.set(ticket, { + kind, binding, expiresAtMs, resolve, @@ -194,15 +227,28 @@ export function createNodeDesktopStreamBroker(deps: { ttlMs?: number; now?: () = }); return { ticket, - attachPath: `${NODE_DESKTOP_ATTACH_PATH}?ticket=${ticket}`, + attachPath: `${kind === "desktop" ? NODE_DESKTOP_ATTACH_PATH : NODE_PORTAL_ATTACH_PATH}?ticket=${ticket}`, expiresAtMs, attached, cancel() { - rejectTicket(ticket, new Error("node desktop stream ticket cancelled")); + rejectTicket(ticket, new Error(`node ${kind} stream ticket cancelled`)); }, }; } + function mint(binding: NodeDesktopStreamBinding) { + return mintStream(binding, "desktop", (stream, metadata) => { + if (!metadata) { + throw new Error("invalid node desktop attach metadata"); + } + return { ...metadata, stream }; + }); + } + + function mintPortal(binding: NodeDesktopStreamBinding) { + return mintStream(binding, "portal", (stream) => ({ stream })); + } + const bindingIsCurrent = async ( registry: TicketNodeRegistry, binding: NodeDesktopStreamBinding, @@ -225,7 +271,13 @@ export function createNodeDesktopStreamBroker(deps: { ttlMs?: number; now?: () = registry: TicketNodeRegistry, ): Promise { const resource = new URL(req.url ?? "/", "http://127.0.0.1"); - if (resource.pathname !== NODE_DESKTOP_ATTACH_PATH) { + const kind = + resource.pathname === NODE_DESKTOP_ATTACH_PATH + ? "desktop" + : resource.pathname === NODE_PORTAL_ATTACH_PATH + ? "portal" + : undefined; + if (!kind) { return false; } const ticket = (resource.searchParams.get("ticket") ?? "").trim(); @@ -234,10 +286,10 @@ export function createNodeDesktopStreamBroker(deps: { ttlMs?: number; now?: () = return true; } const entry = tickets.get(ticket); - if (!entry || entry.redeemed || entry.expiresAtMs <= now()) { + if (!entry || entry.kind !== kind || entry.redeemed || entry.expiresAtMs <= now()) { writeUnauthorized(socket); - if (entry && !entry.redeemed) { - rejectTicket(ticket, new Error("node desktop stream ticket expired")); + if (entry && entry.kind === kind && !entry.redeemed) { + rejectTicket(ticket, new Error(`node ${kind} stream ticket expired`)); } return true; } @@ -245,7 +297,7 @@ export function createNodeDesktopStreamBroker(deps: { ttlMs?: number; now?: () = entry.socket = socket; const onSocketError = (error: Error) => rejectTicket(ticket, error); const onSocketClose = () => - rejectTicket(ticket, new Error("node desktop attach closed during authorization")); + rejectTicket(ticket, new Error(`node ${kind} attach closed during authorization`)); socket.once("error", onSocketError); socket.once("end", onSocketClose); socket.once("close", onSocketClose); @@ -263,7 +315,7 @@ export function createNodeDesktopStreamBroker(deps: { ttlMs?: number; now?: () = socket.off("end", onSocketClose); socket.off("close", onSocketClose); writeUnauthorized(socket); - rejectTicket(ticket, new Error("node desktop stream ticket binding is stale")); + rejectTicket(ticket, new Error(`node ${kind} stream ticket binding is stale`)); return true; } socket.off("error", onSocketError); @@ -273,17 +325,14 @@ export function createNodeDesktopStreamBroker(deps: { ttlMs?: number; now?: () = wss.handleUpgrade(req, socket, head, (ws) => { entry.socket = undefined; entry.ws = ws; - const attached = readAttachedStream(ws, (error) => rejectTicket(ticket, error)); + const attached = readAttachedStream(ws, kind, (error) => rejectTicket(ticket, error)); void (async () => { try { const resolved = await attached; if (!(await bindingIsCurrent(registry, entry.binding))) { - throw new Error("node desktop stream ticket binding is stale"); + throw new Error(`node ${kind} stream ticket binding is stale`); } - resolveTicket(ticket, { - ...resolved.metadata, - stream: resolved.stream, - }); + resolveTicket(ticket, resolved.stream, resolved.metadata); } catch (error) { rejectTicket(ticket, error instanceof Error ? error : new Error(String(error))); } @@ -295,7 +344,7 @@ export function createNodeDesktopStreamBroker(deps: { ttlMs?: number; now?: () = return true; } - return { mint, handleUpgrade }; + return { mint, mintPortal, handleUpgrade }; } export type NodeDesktopStreamBroker = ReturnType; diff --git a/src/gateway/node-runner-inventory-runtime.ts b/src/gateway/node-runner-inventory-runtime.ts index 6994296b19a2..d637789194c6 100644 --- a/src/gateway/node-runner-inventory-runtime.ts +++ b/src/gateway/node-runner-inventory-runtime.ts @@ -110,7 +110,8 @@ export function sameNodeWorkerHostDeclaration( left.capacity.available === right.capacity.available && left.bundlePrewarm === right.bundlePrewarm && left.bundleRetention === right.bundleRetention && - left.bundleStatus === right.bundleStatus)) + left.bundleStatus === right.bundleStatus && + left.portalStream === right.portalStream)) ); } diff --git a/src/gateway/portals/portal-http-proxy.test.ts b/src/gateway/portals/portal-http-proxy.test.ts index 37213d08498d..ef4e1d5e43e4 100644 --- a/src/gateway/portals/portal-http-proxy.test.ts +++ b/src/gateway/portals/portal-http-proxy.test.ts @@ -5,9 +5,11 @@ import { type Server, type ServerResponse, } from "node:http"; -import type { AddressInfo } from "node:net"; +import net, { type AddressInfo } from "node:net"; +import { duplexPair, type Duplex } from "node:stream"; import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; import { type RawData, WebSocket, WebSocketServer } from "ws"; +import type { PortalTarget } from "./portal-http-proxy.js"; import { createGatewayPortalService, type GatewayPortalService } from "./portal-service.js"; type HttpResult = { @@ -25,6 +27,7 @@ const targetServer = createServer((req, res) => targetHandler(req, res)); const targetWss = new WebSocketServer({ server: targetServer }); const services = new Set(); const temporaryTargetServers = new Set(); +const temporaryTargetWebSockets = new Set(); beforeAll(async () => { targetWss.on("connection", (socket, req) => { @@ -47,6 +50,10 @@ beforeAll(async () => { afterEach(async () => { await Promise.all([...services].map((service) => service.closeAll())); services.clear(); + for (const server of temporaryTargetWebSockets) { + server.close(); + } + temporaryTargetWebSockets.clear(); await Promise.all( [...temporaryTargetServers].map( (server) => @@ -78,7 +85,10 @@ function portalService() { async function listenTarget( handler: (req: IncomingMessage, res: ServerResponse) => void, ): Promise { - const server = createServer(handler); + return await listenTargetServer(createServer(handler)); +} + +async function listenTargetServer(server: Server): Promise { temporaryTargetServers.add(server); await new Promise((resolve, reject) => { server.once("error", reject); @@ -87,6 +97,25 @@ async function listenTarget( return (server.address() as AddressInfo).port; } +function createWorkerStream(port: number): Duplex { + const [gatewayStream, workerStream] = duplexPair({ allowHalfOpen: false }); + const appSocket = net.connect({ host: "127.0.0.1", port }); + workerStream.on("error", () => appSocket.destroy()); + appSocket.on("error", () => workerStream.destroy()); + workerStream.pipe(appSocket).pipe(workerStream); + return gatewayStream; +} + +function workerTarget(connect: () => Promise, remotePort: number): PortalTarget { + return { + kind: "worker", + environmentId: "cloud-worker", + ownerEpoch: 3, + remotePort, + connect, + }; +} + async function httpCall(params: { port: number; path?: string; @@ -199,7 +228,7 @@ describe("portal HTTP proxy", () => { res.statusCode = 200; res.end("proxied"); }; - const portal = await portalService().open({ targetPort, title: "App" }); + const portal = (await portalService().open({ targetPort, title: "App" })).portal; const unauthorized = await httpCall({ port: portal.listenPort }); expect(unauthorized.status).toBe(401); @@ -237,8 +266,8 @@ describe("portal HTTP proxy", () => { res.end("target-b"); }); const service = portalService(); - const portalA = await service.open({ targetPort }); - const portalB = await service.open({ targetPort: targetPortB }); + const portalA = (await service.open({ targetPort })).portal; + const portalB = (await service.open({ targetPort: targetPortB })).portal; const jar = new Map(); expect( @@ -292,7 +321,7 @@ describe("portal HTTP proxy", () => { res.write("hello "); res.end("portal"); }; - const portal = await portalService().open({ targetPort }); + const portal = (await portalService().open({ targetPort })).portal; const result = await httpCall({ port: portal.listenPort, path: "/asset?q=1", @@ -338,8 +367,8 @@ describe("portal HTTP proxy", () => { res.end("target-b"); }); const service = portalService(); - const portalA = await service.open({ targetPort }); - const portalB = await service.open({ targetPort: targetPortB }); + const portalA = (await service.open({ targetPort })).portal; + const portalB = (await service.open({ targetPort: targetPortB })).portal; const jar = new Map(); const initialA = await browserCall(jar, { @@ -389,7 +418,7 @@ describe("portal HTTP proxy", () => { res.end("target-a"); }; const service = portalService(); - const portalA = await service.open({ targetPort }); + const portalA = (await service.open({ targetPort })).portal; const jar = new Map(); await browserCall(jar, { port: portalA.listenPort, @@ -405,7 +434,7 @@ describe("portal HTTP proxy", () => { } res.end("target-b"); }; - const portalB = await service.open({ targetPort }); + const portalB = (await service.open({ targetPort })).portal; expect(portalB.tokenQuery).not.toBe(portalA.tokenQuery); await browserCall(jar, { @@ -427,7 +456,7 @@ describe("portal HTTP proxy", () => { res.statusCode = 200; res.end("proxied"); }; - const portal = await portalService().open({ targetPort }); + const portal = (await portalService().open({ targetPort })).portal; const token = portal.tokenQuery.slice("openclaw_portal=".length); const result = await httpCall({ @@ -456,7 +485,7 @@ describe("portal HTTP proxy", () => { res.end(); }); }; - const portal = await portalService().open({ targetPort }); + const portal = (await portalService().open({ targetPort })).portal; const result = await httpCall({ port: portal.listenPort, method: "POST", @@ -480,7 +509,7 @@ describe("portal HTTP proxy", () => { await new Promise((resolve) => { unavailableTarget.close(() => resolve()); }); - const portal = await portalService().open({ targetPort: port }); + const portal = (await portalService().open({ targetPort: port })).portal; const result = await httpCall({ port: portal.listenPort, @@ -491,6 +520,121 @@ describe("portal HTTP proxy", () => { expect(result.body).toContain('http-equiv="refresh" content="2"'); }); + it("proxies worker HTTP and WebSocket traffic through a fresh paired duplex per connection", async () => { + const remotePort = 4173; + let receivedHost: string | undefined; + let receivedWebSocketPath: string | undefined; + let connectionCount = 0; + const appServer = createServer((req, res) => { + receivedHost = req.headers.host; + res.end("worker proxied"); + }); + const socketWss = new WebSocketServer({ server: appServer }); + temporaryTargetWebSockets.add(socketWss); + socketWss.on("connection", (socket, req) => { + receivedWebSocketPath = req.url; + socket.on("message", (data) => socket.send(data)); + }); + const appPort = await listenTargetServer(appServer); + const portal = ( + await portalService().open({ + targetPort: remotePort, + target: workerTarget(async () => { + connectionCount += 1; + return createWorkerStream(appPort); + }, remotePort), + }) + ).portal; + + expect( + await httpCall({ port: portal.listenPort, path: `/preview?${portal.tokenQuery}` }), + ).toMatchObject({ status: 200, body: "worker proxied" }); + expect(receivedHost).toBe(`localhost:${remotePort}`); + + const { socket } = await openWebSocket( + `ws://127.0.0.1:${portal.listenPort}/hmr?${portal.tokenQuery}`, + ); + const echoed = new Promise((resolve) => { + socket.once("message", (data) => resolve(webSocketMessageText(data))); + }); + socket.send("worker hot reload"); + expect(await echoed).toBe("worker hot reload"); + expect(receivedWebSocketPath).toBe("/hmr"); + expect(connectionCount).toBe(2); + await closeWebSocket(socket); + }); + + it("waits for asynchronous worker stream attachment before forwarding the HTTP request", async () => { + targetHandler = (req, res) => { + res.end(`worker ${req.url}`); + }; + let notifyDialStarted!: () => void; + let releaseDial!: () => void; + const dialStarted = new Promise((resolve) => { + notifyDialStarted = resolve; + }); + const dialReleased = new Promise((resolve) => { + releaseDial = resolve; + }); + const remotePort = 4173; + const portal = ( + await portalService().open({ + targetPort: remotePort, + target: workerTarget(async () => { + notifyDialStarted(); + await dialReleased; + return createWorkerStream(targetPort); + }, remotePort), + }) + ).portal; + + const response = httpCall({ + port: portal.listenPort, + path: `/slow?${portal.tokenQuery}`, + }); + await dialStarted; + releaseDial(); + + expect(await response).toMatchObject({ status: 200, body: "worker /slow" }); + }); + + it.each(["rejected", "closed", "reset"] as const)( + "shows the worker retry page for HTTP and upgrades when its node stream is %s", + async (streamState) => { + const remotePort = 4173; + const connect = async (): Promise => { + if (streamState === "rejected") { + throw new Error("Worker node stream unavailable"); + } + const [gatewayStream, workerStream] = duplexPair({ allowHalfOpen: false }); + if (streamState === "closed") { + workerStream.end(); + } else { + workerStream.destroy(); + } + return gatewayStream; + }; + const portal = ( + await portalService().open({ + targetPort: remotePort, + target: workerTarget(connect, remotePort), + }) + ).portal; + const cookie = portalAuthCookie(portal); + const requestHeaders: Record[] = [ + { Cookie: cookie }, + { Cookie: cookie, Connection: "Upgrade", Upgrade: "websocket" }, + ]; + + for (const headers of requestHeaders) { + const result = await httpCall({ port: portal.listenPort, headers }); + expect(result.status).toBe(502); + expect(result.body).toContain(`Waiting for the app on port ${remotePort}…`); + expect(result.body).toContain('http-equiv="refresh" content="2"'); + } + }, + ); + it("reaches IPv6-only targets through the localhost dual-stack dial", async () => { // Node >=17 dev servers (Vite, Next.js) often bind ::1 only on "localhost". const v6Target = createServer((req, res) => { @@ -503,7 +647,7 @@ describe("portal HTTP proxy", () => { }); try { const v6Port = (v6Target.address() as AddressInfo).port; - const portal = await portalService().open({ targetPort: v6Port }); + const portal = (await portalService().open({ targetPort: v6Port })).portal; const result = await httpCall({ port: portal.listenPort, path: `/?${portal.tokenQuery}`, @@ -518,7 +662,7 @@ describe("portal HTTP proxy", () => { it("splices WebSockets and destroys upgraded sockets and listeners on close", async () => { const service = portalService(); - const portal = await service.open({ targetPort }); + const portal = (await service.open({ targetPort })).portal; targetWebSocketSetCookie = "socket=ready; Domain=target.example; Path=/; HttpOnly"; let upgradeCookies: string[] | undefined; const ws = new WebSocket( @@ -562,8 +706,8 @@ describe("portal HTTP proxy", () => { res.end("target-b"); }); const service = portalService(); - const portalA = await service.open({ targetPort }); - const portalB = await service.open({ targetPort: targetPortB }); + const portalA = (await service.open({ targetPort })).portal; + const portalB = (await service.open({ targetPort: targetPortB })).portal; const jar = new Map(); await browserCall(jar, { port: portalA.listenPort, @@ -595,7 +739,7 @@ describe("portal HTTP proxy", () => { it("does not forward WebSocket cookies from a closed portal when its target port is reused", async () => { const service = portalService(); - const portalA = await service.open({ targetPort }); + const portalA = (await service.open({ targetPort })).portal; targetWebSocketSetCookie = "socket=portal-a-secret; Path=/; HttpOnly"; const connectionA = await openWebSocket( `ws://127.0.0.1:${portalA.listenPort}/hmr?${portalA.tokenQuery}`, @@ -605,7 +749,7 @@ describe("portal HTTP proxy", () => { await closeWebSocket(connectionA.socket); await service.close(portalA.id); - const portalB = await service.open({ targetPort }); + const portalB = (await service.open({ targetPort })).portal; targetWebSocketSetCookie = "socket=portal-b; Path=/; HttpOnly"; const connectionB = await openWebSocket( `ws://127.0.0.1:${portalB.listenPort}/hmr?${portalB.tokenQuery}`, diff --git a/src/gateway/portals/portal-http-proxy.ts b/src/gateway/portals/portal-http-proxy.ts index ed0b97677e55..4e032328e3a5 100644 --- a/src/gateway/portals/portal-http-proxy.ts +++ b/src/gateway/portals/portal-http-proxy.ts @@ -6,7 +6,7 @@ import type { ServerResponse, } from "node:http"; import { request as requestHttp } from "node:http"; -import net, { type Socket } from "node:net"; +import net from "node:net"; import type { Duplex } from "node:stream"; const PORTAL_AUTH_NAME = "openclaw_portal"; @@ -36,9 +36,19 @@ const HOP_BY_HOP_HEADERS = new Set([ "upgrade", ]); +export type PortalTarget = + | { kind: "local"; port: number } + | { + kind: "worker"; + environmentId: string; + ownerEpoch: number; + remotePort: number; + connect: () => Promise; + }; + type PortalProxyTarget = { listenPort: number; - targetPort: number; + target: PortalTarget; token: string; cookieNamespace: string; }; @@ -195,11 +205,24 @@ function respondPortalUnauthorized(req: IncomingMessage, res: ServerResponse): v htmlResponse(res, 401, html, req.method === "HEAD"); } -function respondPortalWaiting(req: IncomingMessage, res: ServerResponse, targetPort: number): void { - const html = +function portalWaitingHtml(targetPort: number): string { + return ( '' + - `Waiting for app

Waiting for the app on port ${targetPort}…

`; - htmlResponse(res, 502, html, req.method === "HEAD"); + `Waiting for app

Waiting for the app on port ${targetPort}…

` + ); +} + +function respondPortalWaiting(req: IncomingMessage, res: ServerResponse, targetPort: number): void { + htmlResponse(res, 502, portalWaitingHtml(targetPort), req.method === "HEAD"); +} + +async function connectPortalTarget(target: PortalTarget): Promise { + if (target.kind === "worker") { + return await target.connect(); + } + // Dial "localhost", not a fixed loopback literal: Node >=17 dev servers (Vite, + // Next.js) often bind ::1 only, and family autoselection reaches either stack. + return net.connect({ host: "localhost", autoSelectFamily: true, port: target.port }); } function connectionHeaderTokens(headers: IncomingHttpHeaders): Set { @@ -245,7 +268,7 @@ function proxyHeaders(headers: IncomingHttpHeaders, cookieNamespace?: string): O return result; } -/** Proxies one authorized portal request only to the loopback target. */ +/** Proxies one authorized portal request to its local or worker target. */ export function handlePortalProxyRequest(params: { req: IncomingMessage; res: ServerResponse; @@ -264,44 +287,60 @@ export function handlePortalProxyRequest(params: { const headers = proxyHeaders(req.headers, target.cookieNamespace); const originalHost = req.headers.host; - headers.host = `localhost:${target.targetPort}`; + const targetPort = target.target.kind === "local" ? target.target.port : target.target.remotePort; + headers.host = `localhost:${targetPort}`; headers["x-forwarded-for"] = req.socket.remoteAddress ?? ""; headers["x-forwarded-proto"] = tls ? "https" : "http"; if (originalHost) { headers["x-forwarded-host"] = originalHost; } - // Dial "localhost", not a fixed loopback literal: Node >=17 dev servers (Vite, - // Next.js) often bind ::1 only, and family autoselection reaches either stack. - const proxyReq = requestHttp({ - hostname: "localhost", - createConnection: () => - net.connect({ host: "localhost", autoSelectFamily: true, port: target.targetPort }), - port: target.targetPort, - method: req.method, - path: authorization.requestPath, - headers, - }); - proxyReq.once("response", (proxyRes) => { - for (const [name, value] of Object.entries(proxyHeaders(proxyRes.headers))) { - if (value !== undefined) { - setProxyResponseHeader(res, name, value, target.cookieNamespace); + void connectPortalTarget(target.target).then( + (targetSocket) => { + if (req.aborted || res.destroyed) { + targetSocket.destroy(); + return; } - } - // Overwrite, never default: a target answering with `unsafe-url` would otherwise - // send the token-bearing portal URL to every third-party origin it references. - res.setHeader("Referrer-Policy", PORTAL_REFERRER_POLICY); - res.statusCode = proxyRes.statusCode ?? 502; - proxyRes.pipe(res); - }); - proxyReq.once("error", () => { - if (!res.headersSent) { - respondPortalWaiting(req, res, target.targetPort); - } else { - res.destroy(); - } - }); - req.once("aborted", () => proxyReq.destroy()); - req.pipe(proxyReq); + const proxyReq = requestHttp({ + hostname: "localhost", + createConnection: () => targetSocket, + port: targetPort, + method: req.method, + path: authorization.requestPath, + headers, + }); + proxyReq.once("response", (proxyRes) => { + for (const [name, value] of Object.entries(proxyHeaders(proxyRes.headers))) { + if (value !== undefined) { + setProxyResponseHeader(res, name, value, target.cookieNamespace); + } + } + // Overwrite, never default: a target answering with `unsafe-url` would otherwise + // send the token-bearing portal URL to every third-party origin it references. + res.setHeader("Referrer-Policy", PORTAL_REFERRER_POLICY); + res.statusCode = proxyRes.statusCode ?? 502; + proxyRes.pipe(res); + }); + proxyReq.once("error", () => { + if (!res.headersSent) { + respondPortalWaiting(req, res, targetPort); + } else { + res.destroy(); + } + }); + proxyReq.once("close", () => { + if (target.target.kind === "worker" && !res.headersSent && !res.writableEnded) { + respondPortalWaiting(req, res, targetPort); + } + }); + req.once("aborted", () => proxyReq.destroy()); + req.pipe(proxyReq); + }, + () => { + if (!res.headersSent && !res.writableEnded && !res.destroyed) { + respondPortalWaiting(req, res, targetPort); + } + }, + ); } function websocketHeaders( @@ -350,10 +389,20 @@ function rejectPortalUpgrade(socket: Duplex): void { ); } +function respondUpgradeWaiting(socket: Duplex, targetPort: number): void { + const html = portalWaitingHtml(targetPort); + socket.end( + "HTTP/1.1 502 Bad Gateway\r\nContent-Type: text/html; charset=utf-8\r\n" + + `Cache-Control: no-store\r\nReferrer-Policy: ${PORTAL_REFERRER_POLICY}\r\n` + + `Content-Length: ${Buffer.byteLength(html)}\r\nConnection: close\r\n\r\n${html}`, + ); +} + function forwardWebSocketResponse( - targetSocket: Socket, + targetSocket: Duplex, browserSocket: Duplex, cookieNamespace: string, + onResponse: () => void, ): void { let pending = Buffer.alloc(0); const onData = (chunk: Buffer) => { @@ -377,6 +426,7 @@ function forwardWebSocketResponse( const rewritten = rewriteTargetCookie(line.slice(separator + 1).trimStart(), cookieNamespace); return rewritten ? [`${line.slice(0, separator)}: ${rewritten}`] : []; }); + onResponse(); browserSocket.write(`${rewrittenLines.join("\r\n")}\r\n\r\n`); const remainder = pending.subarray(headerEnd + 4); if (remainder.length > 0) { @@ -387,7 +437,7 @@ function forwardWebSocketResponse( targetSocket.on("data", onData); } -/** Splices an authorized portal WebSocket upgrade into the loopback target. */ +/** Splices an authorized portal WebSocket upgrade into its local or worker target. */ export function handlePortalProxyUpgrade(params: { req: IncomingMessage; socket: Duplex; @@ -402,33 +452,62 @@ export function handlePortalProxyUpgrade(params: { return; } - // Same localhost/dual-stack contract as the HTTP path above. - const targetSocket: Socket = net.connect({ - host: "localhost", - autoSelectFamily: true, - port: target.targetPort, - }); + const targetPort = target.target.kind === "local" ? target.target.port : target.target.remotePort; upgradedSockets.add(socket); - upgradedSockets.add(targetSocket); - const release = (stream: Duplex) => upgradedSockets.delete(stream); - socket.once("close", () => { - release(socket); - targetSocket.destroy(); - }); - targetSocket.once("close", () => { - release(targetSocket); - socket.destroy(); - }); - socket.once("error", () => targetSocket.destroy()); - targetSocket.once("error", () => socket.destroy()); - targetSocket.once("connect", () => { - forwardWebSocketResponse(targetSocket, socket, target.cookieNamespace); - targetSocket.write( - websocketHeaders(req, target.targetPort, target.cookieNamespace, authorization.requestPath), - ); - if (head.length > 0) { - targetSocket.write(head); - } - socket.pipe(targetSocket); - }); + socket.once("close", () => upgradedSockets.delete(socket)); + void connectPortalTarget(target.target).then( + (targetSocket) => { + if (socket.destroyed) { + targetSocket.destroy(); + return; + } + upgradedSockets.add(targetSocket); + let responseStarted = false; + let waitingResponseSent = false; + const closeUpgrade = () => { + if (target.target.kind === "worker" && !responseStarted && !socket.destroyed) { + if (!waitingResponseSent) { + waitingResponseSent = true; + respondUpgradeWaiting(socket, targetPort); + } + return; + } + socket.destroy(); + }; + socket.once("close", () => targetSocket.destroy()); + targetSocket.once("close", () => { + upgradedSockets.delete(targetSocket); + closeUpgrade(); + }); + targetSocket.once("end", closeUpgrade); + socket.once("error", () => targetSocket.destroy()); + targetSocket.once("error", closeUpgrade); + const spliceUpgrade = () => { + forwardWebSocketResponse(targetSocket, socket, target.cookieNamespace, () => { + responseStarted = true; + }); + targetSocket.write( + websocketHeaders(req, targetPort, target.cookieNamespace, authorization.requestPath), + ); + if (head.length > 0) { + targetSocket.write(head); + } + socket.pipe(targetSocket); + }; + if (target.target.kind === "worker") { + spliceUpgrade(); + } else { + targetSocket.once("connect", spliceUpgrade); + } + }, + () => { + if (!socket.destroyed) { + if (target.target.kind === "worker") { + respondUpgradeWaiting(socket, targetPort); + } else { + socket.destroy(); + } + } + }, + ); } diff --git a/src/gateway/portals/portal-service.test.ts b/src/gateway/portals/portal-service.test.ts index fe71e2527ed1..d547be074172 100644 --- a/src/gateway/portals/portal-service.test.ts +++ b/src/gateway/portals/portal-service.test.ts @@ -1,4 +1,5 @@ import { request, type Server } from "node:http"; +import type { Duplex } from "node:stream"; import { afterEach, describe, expect, it, vi } from "vitest"; import { getFreePort } from "../../test-utils/ports.js"; import * as httpListen from "../server/http-listen.js"; @@ -6,6 +7,10 @@ import { createGatewayPortalService, type GatewayPortalService } from "./portal- const services = new Set(); +async function unavailableWorkerConnection(): Promise { + throw new Error("Worker connection unavailable"); +} + afterEach(async () => { vi.restoreAllMocks(); await Promise.all([...services].map((service) => service.closeAll())); @@ -40,10 +45,28 @@ async function getDistinctFreePort(excluded: number): Promise { throw new Error("Failed to reserve a distinct test port"); } +describe("portal open authority fence", () => { + it("refuses to mutate a reused portal when the caller's authority lapsed", async () => { + const { service } = makeService(["127.0.0.1"]); + const first = (await service.open({ targetPort: 41234, title: "Live" })).portal; + await expect( + service.open({ + targetPort: 41234, + title: "Hijacked", + assertCurrent: () => { + throw new Error("authority lapsed"); + }, + }), + ).rejects.toThrow("authority lapsed"); + const summary = service.list().find((portal) => portal.id === first.id); + expect(summary?.title).toBe("Live"); + }); +}); + describe("gateway portal service", () => { it("allocates one port across every frozen bind host", async () => { const { service, httpServers } = makeService(["127.0.0.1", "::1"]); - const portal = await service.open({ targetPort: 3000, title: "App" }); + const portal = (await service.open({ targetPort: 3000, title: "App" })).portal; expect(portal).toMatchObject({ id: "p3000", port: 3000, title: "App" }); expect(portal.listenPort).toBeGreaterThan(0); @@ -72,7 +95,7 @@ describe("gateway portal service", () => { }); const { service, httpServers } = makeService(["127.0.0.1", "::1"]); - const portal = await service.open({ targetPort }); + const portal = (await service.open({ targetPort })).portal; expect(portal.listenPort).toBe(acceptedPort); expect(calls).toEqual([ @@ -118,13 +141,15 @@ describe("gateway portal service", () => { it("updates an existing target without replacing its listener or token", async () => { const { service, httpServers } = makeService(["127.0.0.1"]); - const first = await service.open({ targetPort: 3000, title: "First" }); - const second = await service.open({ - targetPort: 3000, - title: "Second", - description: "Updated", - path: "/preview", - }); + const first = (await service.open({ targetPort: 3000, title: "First" })).portal; + const second = ( + await service.open({ + targetPort: 3000, + title: "Second", + description: "Updated", + path: "/preview", + }) + ).portal; expect(second).toMatchObject({ id: first.id, @@ -140,11 +165,181 @@ describe("gateway portal service", () => { expect(service.list()).toEqual([second]); }); + it("keeps local and worker portals on the same application port distinct", async () => { + const { service } = makeService(["127.0.0.1"]); + const local = (await service.open({ targetPort: 3000 })).portal; + const worker = ( + await service.open({ + targetPort: 3000, + target: { + kind: "worker", + environmentId: "cloud/a", + ownerEpoch: 7, + remotePort: 3000, + connect: unavailableWorkerConnection, + }, + origin: "Cloud worker A", + }) + ).portal; + const otherWorker = ( + await service.open({ + targetPort: 3000, + target: { + kind: "worker", + environmentId: "cloud-a", + ownerEpoch: 7, + remotePort: 3000, + connect: unavailableWorkerConnection, + }, + }) + ).portal; + const staleWorker = ( + await service.open({ + targetPort: 3000, + target: { + kind: "worker", + environmentId: "cloud/a", + ownerEpoch: 6, + remotePort: 3000, + connect: unavailableWorkerConnection, + }, + }) + ).portal; + + expect(local.id).toBe("p3000"); + expect(new Set([local.id, worker.id, otherWorker.id, staleWorker.id]).size).toBe(4); + expect(worker).toMatchObject({ port: 3000, origin: "Cloud worker A" }); + expect(service.list()).toHaveLength(4); + expect(service.listWorkerPortals("cloud/a", 7)).toEqual([worker]); + expect(service.listWorkerPortals("cloud/a", 6)).toEqual([staleWorker]); + expect(service.listWorkerPortals("cloud-a", 7)).toEqual([otherWorker]); + expect(service.listWorkerPortals("cloud/a", 8)).toEqual([]); + }); + + it("closes worker forwards only for the selected environment owner epoch", async () => { + const { service } = makeService(["127.0.0.1"]); + const closeStaleForward = vi.fn(); + const closeCurrentForward = vi.fn(); + const stale = ( + await service.open({ + targetPort: 3000, + target: { + kind: "worker", + environmentId: "cloud-a", + ownerEpoch: 6, + remotePort: 3000, + connect: unavailableWorkerConnection, + }, + onClose: closeStaleForward, + }) + ).portal; + const current = ( + await service.open({ + targetPort: 3000, + target: { + kind: "worker", + environmentId: "cloud-a", + ownerEpoch: 7, + remotePort: 3000, + connect: unavailableWorkerConnection, + }, + onClose: closeCurrentForward, + }) + ).portal; + + await service.closeWorkerPortals("cloud-a", 6); + + expect(closeStaleForward).toHaveBeenCalledOnce(); + expect(closeCurrentForward).not.toHaveBeenCalled(); + expect(service.list().map((portal) => portal.id)).toEqual([current.id]); + expect(stale.id).not.toBe(current.id); + + await service.close(current.id); + expect(closeCurrentForward).toHaveBeenCalledOnce(); + }); + + it("keeps worker portal ids bounded for the longest supported environment id", async () => { + const { service } = makeService(["127.0.0.1"]); + const environmentId = "w".repeat(256); + const portal = ( + await service.open({ + targetPort: 3000, + target: { + kind: "worker", + environmentId, + ownerEpoch: 7, + remotePort: 3000, + connect: unavailableWorkerConnection, + }, + }) + ).portal; + + expect(portal.id.length).toBeLessThanOrEqual(256); + expect(service.listWorkerPortals(environmentId, 7)).toEqual([portal]); + await service.close(portal.id); + expect(service.list()).toEqual([]); + }); + + it("revalidates worker close authority immediately before queued removal", async () => { + const { service } = makeService(["127.0.0.1"]); + const portal = (await service.open({ targetPort: 3000 })).portal; + let authorityCurrent = true; + const assertCurrent = () => { + if (!authorityCurrent) { + throw new Error("Worker portal authority changed"); + } + }; + const closing = service.close(portal.id, assertCurrent); + authorityCurrent = false; + + await expect(closing).rejects.toThrow("Worker portal authority changed"); + expect(service.list()).toEqual([portal]); + }); + + it("fences a worker portal whose listener is still opening during owner teardown", async () => { + const actualListen = httpListen.listenGatewayHttpServer; + let notifyBindStarted: (() => void) | undefined; + let releaseBind: (() => void) | undefined; + const bindStarted = new Promise((resolve) => { + notifyBindStarted = resolve; + }); + const bindReleased = new Promise((resolve) => { + releaseBind = resolve; + }); + vi.spyOn(httpListen, "listenGatewayHttpServer").mockImplementation(async (params) => { + notifyBindStarted?.(); + await bindReleased; + await actualListen(params); + }); + const { service } = makeService(["127.0.0.1"]); + const closeForward = vi.fn(); + const opening = service.open({ + targetPort: 3000, + target: { + kind: "worker", + environmentId: "cloud-a", + ownerEpoch: 7, + remotePort: 3000, + connect: unavailableWorkerConnection, + }, + onClose: closeForward, + }); + await bindStarted; + + const closing = service.closeWorkerPortals("cloud-a", 7); + releaseBind?.(); + await opening; + await closing; + + expect(service.list()).toEqual([]); + expect(closeForward).toHaveBeenCalledOnce(); + }); + it("closes idempotently and closes every portal on shutdown", async () => { const { service, httpServers } = makeService(["127.0.0.1"]); - const first = await service.open({ targetPort: 3000 }); + const first = (await service.open({ targetPort: 3000 })).portal; const firstServer = httpServers.at(-1); - const second = await service.open({ targetPort: 4000 }); + const second = (await service.open({ targetPort: 4000 })).portal; const secondServer = httpServers.at(-1); expect(firstServer).toBeDefined(); expect(secondServer).toBeDefined(); @@ -177,7 +372,7 @@ describe("gateway portal service", () => { ["::", "[::1]"], ])("maps wildcard bind host %s to openable host %s", async (bindHost, openableHost) => { const { service } = makeService([bindHost]); - const portal = await service.open({ targetPort: 3000 }); + const portal = (await service.open({ targetPort: 3000 })).portal; expect(portal.publicUrl).toBe(`http://${openableHost}:${portal.listenPort}/`); expect(portal.url).toBe(`${portal.publicUrl}?${portal.tokenQuery}`); diff --git a/src/gateway/portals/portal-service.ts b/src/gateway/portals/portal-service.ts index 468850b3fbde..875b488d2be6 100644 --- a/src/gateway/portals/portal-service.ts +++ b/src/gateway/portals/portal-service.ts @@ -8,8 +8,13 @@ import type { PortalOpenResult, PortalSummary, } from "../../../packages/gateway-protocol/src/index.js"; +import { sha256HexPrefixCore } from "../../infra/crypto-digest.js"; import { listenGatewayHttpServer } from "../server/http-listen.js"; -import { handlePortalProxyRequest, handlePortalProxyUpgrade } from "./portal-http-proxy.js"; +import { + handlePortalProxyRequest, + handlePortalProxyUpgrade, + type PortalTarget, +} from "./portal-http-proxy.js"; const PORTAL_PORT_ALLOCATION_ATTEMPTS = 10; @@ -18,7 +23,8 @@ type PortalEntry = { title: string; description?: string; path?: string; - targetPort: number; + origin?: string; + target: PortalTarget; token: string; cookieNamespace: string; listenPort: number; @@ -29,19 +35,30 @@ type PortalRuntimeEntry = { portal: PortalEntry; servers: HttpServer[]; upgradedSockets: Set; + onClose?: () => Promise | void; }; type GatewayPortalOpenParams = { targetPort: number; + target?: PortalTarget; + /** Revalidated inside the serialized operation; reuse must not mutate a live portal for a caller whose authority lapsed. */ + assertCurrent?: () => void; + onClose?: () => Promise | void; + origin?: string; title?: string; description?: string; path?: string; }; export type GatewayPortalService = { - open: (params: GatewayPortalOpenParams) => Promise; + /** `created` is false when an already-registered portal was reused; its target and onClose are unchanged. */ + open: ( + params: GatewayPortalOpenParams, + ) => Promise<{ portal: PortalOpenResult; created: boolean }>; list: () => PortalSummary[]; - close: (id: string) => Promise; + listWorkerPortals: (environmentId: string, ownerEpoch: number) => PortalSummary[]; + close: (id: string, assertCurrent?: () => void) => Promise; + closeWorkerPortals: (environmentId: string, ownerEpoch?: number) => Promise; closeAll: () => Promise; }; @@ -98,13 +115,14 @@ export function createGatewayPortalService(params: { return { id: portal.id, title: portal.title, - port: portal.targetPort, + port: portal.target.kind === "local" ? portal.target.port : portal.target.remotePort, listenPort: portal.listenPort, tokenQuery, url: openableUrl.toString(), publicUrl, ...(portal.path ? { path: portal.path } : {}), ...(portal.description ? { description: portal.description } : {}), + ...(portal.origin ? { origin: portal.origin } : {}), createdAtMs: portal.createdAtMs, }; }; @@ -139,15 +157,27 @@ export function createGatewayPortalService(params: { } runtime.upgradedSockets.clear(); await closeServers(runtime.servers); + await runtime.onClose?.(); }; + const summarizeEntries = (selected: Iterable): PortalSummary[] => + Array.from(selected, ({ portal }) => summarize(portal)).toSorted( + (left, right) => left.createdAtMs - right.createdAtMs || left.id.localeCompare(right.id), + ); + return { open: async (input) => { - const id = `p${input.targetPort}`; + const target: PortalTarget = input.target ?? { kind: "local", port: input.targetPort }; + const targetPort = target.kind === "local" ? target.port : target.remotePort; + const id = + target.kind === "local" + ? `p${targetPort}` + : `p${targetPort}-worker-${sha256HexPrefixCore(target.environmentId, 32)}-${target.ownerEpoch}`; return await serialize(id, async () => { if (closed) { throw new Error("portals unavailable"); } + input.assertCurrent?.(); const existing = entries.get(id); if (existing) { existing.portal.title = input.title?.trim() || existing.portal.title; @@ -157,7 +187,10 @@ export function createGatewayPortalService(params: { if (input.path !== undefined) { existing.portal.path = input.path; } - return summarize(existing.portal); + if (input.origin !== undefined) { + existing.portal.origin = input.origin; + } + return { portal: summarize(existing.portal), created: false }; } if (params.httpBindHosts.length === 0) { throw new Error("Gateway listener must start before opening a portal"); @@ -165,10 +198,11 @@ export function createGatewayPortalService(params: { const portal: PortalEntry = { id, - title: input.title?.trim() || `Port ${input.targetPort}`, + title: input.title?.trim() || `Port ${targetPort}`, ...(input.description ? { description: input.description } : {}), ...(input.path ? { path: input.path } : {}), - targetPort: input.targetPort, + ...(input.origin ? { origin: input.origin } : {}), + target, token: randomBytes(32).toString("hex"), cookieNamespace: randomBytes(16).toString("hex"), listenPort: 0, @@ -211,7 +245,7 @@ export function createGatewayPortalService(params: { if (!address || typeof address === "string") { throw new Error("Portal listener failed to resolve its port"); } - if (address.port !== portal.targetPort) { + if (target.kind === "worker" || address.port !== targetPort) { portal.listenPort = address.port; break; } @@ -219,9 +253,7 @@ export function createGatewayPortalService(params: { await closeServers([primaryServer]); } if (portal.listenPort === 0) { - throw new Error( - `Portal listener repeatedly allocated target port ${portal.targetPort}`, - ); + throw new Error(`Portal listener repeatedly allocated target port ${targetPort}`); } for (const [index, host] of params.httpBindHosts.entries()) { if (index === 0) { @@ -243,20 +275,46 @@ export function createGatewayPortalService(params: { } catch (error) { removeServers(params.httpServers, servers); await closeServers(servers); + await input.onClose?.(); throw error; } - entries.set(id, { portal, servers, upgradedSockets }); - return summarize(portal); + entries.set(id, { + portal, + servers, + upgradedSockets, + ...(input.onClose ? { onClose: input.onClose } : {}), + }); + return { portal: summarize(portal), created: true }; }); }, - list: () => - [...entries.values()] - .map(({ portal }) => summarize(portal)) - .toSorted( - (left, right) => left.createdAtMs - right.createdAtMs || left.id.localeCompare(right.id), + list: () => summarizeEntries(entries.values()), + listWorkerPortals: (environmentId, ownerEpoch) => + summarizeEntries( + [...entries.values()].filter( + ({ portal }) => + portal.target.kind === "worker" && + portal.target.environmentId === environmentId && + portal.target.ownerEpoch === ownerEpoch, ), - close: async (id) => { - await serialize(id, () => closeEntry(id)); + ), + close: async (id, assertCurrent) => { + await serialize(id, () => { + assertCurrent?.(); + return closeEntry(id); + }); + }, + closeWorkerPortals: async (environmentId, ownerEpoch) => { + const environmentSuffix = `-worker-${sha256HexPrefixCore(environmentId, 32)}-`; + // Include in-flight opens so teardown fences a listener still awaiting its bind. + const ids = [...new Set([...entries.keys(), ...operations.keys()])].filter((id) => { + const separator = id.indexOf(environmentSuffix); + return ( + separator >= 0 && + (ownerEpoch === undefined || + id.slice(separator + environmentSuffix.length) === String(ownerEpoch)) + ); + }); + await Promise.all(ids.map((id) => serialize(id, () => closeEntry(id)))); }, closeAll: async () => { closed = true; diff --git a/src/gateway/server-http-upgrades.ts b/src/gateway/server-http-upgrades.ts index 3830d71f8711..0ceff7a0e68f 100644 --- a/src/gateway/server-http-upgrades.ts +++ b/src/gateway/server-http-upgrades.ts @@ -11,7 +11,10 @@ import { isGatewayWorkAdmissionClosed, } from "../process/gateway-work-admission.js"; import { createLazyRuntimeModule } from "../shared/lazy-runtime.js"; -import { NODE_DESKTOP_ATTACH_PATH } from "../shared/node-desktop-stream.js"; +import { + NODE_DESKTOP_ATTACH_PATH, + NODE_PORTAL_ATTACH_PATH, +} from "../shared/node-desktop-stream.js"; import { AUTH_RATE_LIMIT_SCOPE_WORKER_ADMISSION, type AuthRateLimiter } from "./auth-rate-limit.js"; import type { GatewayAuthResult, ResolvedGatewayAuth } from "./auth.js"; import type { NodeDesktopStreamBroker } from "./desktop/node-stream-broker.js"; @@ -410,10 +413,11 @@ export function attachGatewayUpgradeHandler(opts: { }); return; } - if (requestPath === NODE_DESKTOP_ATTACH_PATH) { + if (requestPath === NODE_DESKTOP_ATTACH_PATH || requestPath === NODE_PORTAL_ATTACH_PATH) { const context = opts.getGatewayRequestContext?.(); if (!opts.nodeDesktopStreamBroker || !context) { - writeGatewayUpgradeServiceUnavailable(socket, "node desktop attach unavailable"); + const feature = requestPath === NODE_DESKTOP_ATTACH_PATH ? "desktop" : "portal"; + writeGatewayUpgradeServiceUnavailable(socket, `node ${feature} attach unavailable`); socket.destroy(); return; } diff --git a/src/gateway/server-methods/nodes.runner-inventory.test.ts b/src/gateway/server-methods/nodes.runner-inventory.test.ts index 3865e90b1859..11b48aa977ab 100644 --- a/src/gateway/server-methods/nodes.runner-inventory.test.ts +++ b/src/gateway/server-methods/nodes.runner-inventory.test.ts @@ -292,6 +292,42 @@ describe("nodeHandlers node.runnerInventory.update", () => { runtime.nodeRegistry.unregister("conn-1"); }); + it("publishes and retires negotiated portal-stream capability without exposing a private command", async () => { + const inventoryChanged = vi.fn(); + const runtime = createNodeRegistryRuntime(() => new NodeRegistry()); + setNodeRunnerStateChangedListener(runtime.nodeRegistry, inventoryChanged); + const client = createWorkerSupervisorNodeClient(); + runtime.nodeRegistry.register(client, { + pairingIdentity: "identity-1", + pairingGeneration: "generation-1", + }); + const publish = async (portalStream: boolean) => { + await runnerInventoryHandler( + runnerInventoryOptions({ + nodeRegistry: runtime.nodeRegistry, + client, + declaration: { + ...availableHost, + workerHost: { + ...availableHost.workerHost, + ...(portalStream ? { portalStream: 1 } : {}), + }, + }, + }), + ); + const [proof] = await runtime.nodeWorkerSupervisorTransport.listCurrentNodes(); + return proof; + }; + + expect((await publish(false))?.workerHost.portalStream).toBeUndefined(); + const supported = await publish(true); + expect(supported?.workerHost.portalStream).toBe(1); + expect(supported?.commands).not.toContain("worker.portal.stream.v1"); + expect((await publish(false))?.workerHost.portalStream).toBeUndefined(); + expect(inventoryChanged).toHaveBeenCalledTimes(3); + runtime.nodeRegistry.unregister("conn-1"); + }); + it("requires a fresh current-generation publication after same-connection promotion", async () => { const runtime = createNodeRegistryRuntime(() => new NodeRegistry()); const client = createWorkerSupervisorNodeClient(); @@ -683,6 +719,13 @@ describe("nodeHandlers node.runnerInventory.update", () => { workerHost: { enabled: true, capacity: AVAILABLE_CAPACITY, bundleStatus: 2 }, }, }, + { + name: "unsupported portal stream version", + params: { + protocolFeatures: [NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE], + workerHost: { enabled: true, capacity: AVAILABLE_CAPACITY, portalStream: 2 }, + }, + }, { name: "bundle status without bundle retention", params: { diff --git a/src/gateway/server-methods/portals.test.ts b/src/gateway/server-methods/portals.test.ts index 98ce387845f4..d607b2828ee0 100644 --- a/src/gateway/server-methods/portals.test.ts +++ b/src/gateway/server-methods/portals.test.ts @@ -47,13 +47,15 @@ describe("portal gateway methods", () => { let portals: PortalSummary[] = []; const service: GatewayPortalService = { list: () => portals, + listWorkerPortals: () => [], open: vi.fn(async () => { portals = [portal]; - return portal; + return { portal, created: true }; }), close: vi.fn(async () => { portals = []; }), + closeWorkerPortals: vi.fn(async () => {}), closeAll: vi.fn(async () => {}), }; const { invoke, broadcast } = harness(service); @@ -100,8 +102,10 @@ describe("portal gateway methods", () => { it("returns portal credentials only to write-capable operators", async () => { const service: GatewayPortalService = { list: () => [portal], + listWorkerPortals: () => [], open: vi.fn(), close: vi.fn(), + closeWorkerPortals: vi.fn(), closeAll: vi.fn(), }; @@ -129,8 +133,10 @@ describe("portal gateway methods", () => { it("rejects malformed requests before service access and reports absent transports", async () => { const service: GatewayPortalService = { list: vi.fn(() => []), + listWorkerPortals: vi.fn(() => []), open: vi.fn(), close: vi.fn(), + closeWorkerPortals: vi.fn(), closeAll: vi.fn(), }; const invalid = await harness(service).invoke("portal.open", { port: 0 }); @@ -152,10 +158,12 @@ describe("portal gateway methods", () => { it("returns Error messages without the Error prefix", async () => { const service: GatewayPortalService = { list: () => [], + listWorkerPortals: () => [], open: vi.fn(async () => { throw new Error("portal bind failed"); }), close: vi.fn(async () => {}), + closeWorkerPortals: vi.fn(async () => {}), closeAll: vi.fn(async () => {}), }; diff --git a/src/gateway/server-methods/portals.ts b/src/gateway/server-methods/portals.ts index 309b8daa5949..d74dfde72fb1 100644 --- a/src/gateway/server-methods/portals.ts +++ b/src/gateway/server-methods/portals.ts @@ -58,7 +58,7 @@ export const portalHandlers: GatewayRequestHandlers = { return; } try { - const portal = await service.open({ + const opened = await service.open({ targetPort: request.port, ...(request.title !== undefined ? { title: request.title } : {}), ...(request.description !== undefined ? { description: request.description } : {}), @@ -69,7 +69,7 @@ export const portalHandlers: GatewayRequestHandlers = { { portals: service.list().map(redactPortalSummary) }, { dropIfSlow: true }, ); - respond(true, portal, undefined); + respond(true, opened.portal, undefined); } catch (error) { respond( false, diff --git a/src/gateway/server-runtime-state-prepare.ts b/src/gateway/server-runtime-state-prepare.ts index 8a1382c0ad7d..37ef8eba1820 100644 --- a/src/gateway/server-runtime-state-prepare.ts +++ b/src/gateway/server-runtime-state-prepare.ts @@ -130,7 +130,7 @@ export async function prepareGatewayKernelState(params: { ? createDesktopSessionRegistry() : undefined; const nodeDesktopStreamBroker = - nodeDesktopObserveAvailable || workerDesktopObserveAvailable + nodeDesktopObserveAvailable || shouldStartWorkerEnvironmentService ? ( await startupTrace.measure( "node-desktop.runtime-import", @@ -156,6 +156,7 @@ export async function prepareGatewayKernelState(params: { const workerModule = await loadWorkerEnvironmentStartupModule(); return await workerModule.createGatewayWorkerEnvironmentRuntime({ getPluginRegistry: () => pluginRuntime.registry, + getPortalRuntime: () => pluginGatewayContext.current, desktopSessionRegistry, ...(nodeDesktopStreamBroker ? { nodeDesktopStreamBroker } : {}), startup: workerEnvironmentStartup, diff --git a/src/gateway/server-worker-environment-startup.test.ts b/src/gateway/server-worker-environment-startup.test.ts index e38e667e760a..f65bd3bcd40b 100644 --- a/src/gateway/server-worker-environment-startup.test.ts +++ b/src/gateway/server-worker-environment-startup.test.ts @@ -4,7 +4,10 @@ import { afterEach, describe, expect, it } from "vitest"; import { GATEWAY_CLIENT_IDS } from "../../packages/gateway-protocol/src/client-info.js"; import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; import { resetConfigRuntimeState, setRuntimeConfigSnapshot } from "../config/config.js"; -import { NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE } from "../infra/node-runner-inventory.js"; +import { + NODE_WORKER_PORTAL_STREAM_VERSION, + NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE, +} from "../infra/node-runner-inventory.js"; import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; import { withEnvAsync } from "../test-utils/env.js"; import { createNodeDesktopStreamBroker } from "./desktop/node-stream-broker.js"; @@ -43,6 +46,7 @@ describe("gateway worker environment startup", () => { const startup = await loadGatewayWorkerEnvironmentStartupState(); const runtime = await createGatewayWorkerEnvironmentRuntime({ getPluginRegistry: () => ({ workerProviders: new Map() }), + getPortalRuntime: () => undefined, desktopSessionRegistry: createDesktopSessionRegistry({ lingerMs: 1 }), startup, log: { child: () => ({ warn: () => {} }) }, @@ -103,6 +107,7 @@ describe("gateway worker environment startup", () => { const runtime = await createGatewayWorkerEnvironmentRuntime({ getPluginRegistry: () => ({ workerProviders: new Map() }), + getPortalRuntime: () => undefined, desktopSessionRegistry: createDesktopSessionRegistry({ lingerMs: 1 }), startup, log: { child: () => ({ warn: () => {} }) }, @@ -191,6 +196,7 @@ describe("gateway worker environment startup", () => { }; const runtime = await createGatewayWorkerEnvironmentRuntime({ getPluginRegistry: () => ({ workerProviders: new Map() }), + getPortalRuntime: () => undefined, desktopSessionRegistry: createDesktopSessionRegistry({ lingerMs: 1 }), nodeDesktopStreamBroker: createNodeDesktopStreamBroker(), startup, @@ -202,6 +208,21 @@ describe("gateway worker environment startup", () => { } runtime.bindWorkerNodeDesktopControl(transport); try { + await expect( + service.supportsNodePortal(record.environmentId, record.ownerEpoch), + ).resolves.toBe(false); + runtime.bindDeviceNodeControl?.(transport); + await expect( + service.supportsNodePortal(record.environmentId, record.ownerEpoch), + ).resolves.toBe(false); + proof.workerHost.portalStream = NODE_WORKER_PORTAL_STREAM_VERSION; + await expect( + service.supportsNodePortal(record.environmentId, record.ownerEpoch), + ).resolves.toBe(true); + delete proof.workerHost.portalStream; + await expect( + service.supportsNodePortal(record.environmentId, record.ownerEpoch), + ).resolves.toBe(false); await expect( service.launchDesktopApp({ environmentId: record.environmentId, app: "terminal" }), ).resolves.toEqual({ app: "terminal", status: "ready" }); diff --git a/src/gateway/server-worker-environment-startup.ts b/src/gateway/server-worker-environment-startup.ts index feab801a7342..2b4ba0f94794 100644 --- a/src/gateway/server-worker-environment-startup.ts +++ b/src/gateway/server-worker-environment-startup.ts @@ -13,6 +13,7 @@ import type { NodeDesktopStreamBroker } from "./desktop/node-stream-broker.js"; import type { DesktopSessionRegistry } from "./desktop/session-registry.js"; import type { GitHubPublicationCoordinator } from "./github-publication.js"; import type { NodeWorkerSupervisorTransport } from "./node-registry-private.js"; +import type { GatewayRequestContext } from "./server-methods/types.js"; import type { WorkerBundleProducer, WorkerNpmArtifact } from "./worker-environments/bundle.js"; import { bindDeviceWorkerAvailability, @@ -108,6 +109,7 @@ export async function loadGatewayWorkerEnvironmentStartupState(): Promise Pick; + getPortalRuntime: () => Pick | undefined; desktopSessionRegistry: DesktopSessionRegistry; nodeDesktopStreamBroker?: NodeDesktopStreamBroker; startup: GatewayWorkerEnvironmentStartupState; @@ -128,6 +130,7 @@ export async function createGatewayWorkerEnvironmentRuntime(params: { { createNodeWorkspaceTransferHttpCallback }, { createWorkerSessionToolExecutor }, { createWorkerNodeDesktopCarrier }, + { createWorkerNodePortalCarrier }, { resolveWorkerProvider }, ] = await Promise.all([ import("./worker-environments/service.js"), @@ -143,6 +146,7 @@ export async function createGatewayWorkerEnvironmentRuntime(params: { import("./worker-environments/node-workspace-transfer-http.js"), import("./worker-environments/worker-session-tool-executor.js"), import("./worker-environments/node-desktop-carrier.js"), + import("./worker-environments/portal-node-carrier.js"), import("../plugins/worker-provider-registry.js"), ]); // The Gateway state-directory lock proves that executors from the previous @@ -210,7 +214,22 @@ export async function createGatewayWorkerEnvironmentRuntime(params: { const workerTunnelManager = createWorkerTunnelManager({ desktopSessionRegistry: params.desktopSessionRegistry, }); + const notifyPortalChange = () => { + const runtime = params.getPortalRuntime(); + const service = runtime?.portalService; + if (!service) { + return; + } + runtime.broadcast( + "portal.changed", + { + portals: service.list().map(({ tokenQuery: _tokenQuery, url: _url, ...portal }) => portal), + }, + { dropIfSlow: true }, + ); + }; const workerNodeDesktopStreamBroker = params.nodeDesktopStreamBroker; + const workerNodePortalCarrier = createWorkerNodePortalCarrier({ store: params.startup.store }); const workerNodeDesktopCarrier = workerNodeDesktopStreamBroker ? createWorkerNodeDesktopCarrier({ store: params.startup.store, @@ -276,6 +295,15 @@ export async function createGatewayWorkerEnvironmentRuntime(params: { tunnelManager: workerTunnelManager, nodeTunnelManager: nodeWorkerTunnelManager, nodeDesktopCarrier: workerNodeDesktopCarrier, + nodePortalCarrier: workerNodePortalCarrier, + closeWorkerPortals: async (environmentId, ownerEpoch) => { + const service = params.getPortalRuntime()?.portalService; + if (!service) { + return; + } + await service.closeWorkerPortals(environmentId, ownerEpoch); + notifyPortalChange(); + }, stopNodeWorkerBundleTransfers: () => nodeWorkerBundleTransfer.closeAll(), applyTranscriptCommit: createWorkerTranscriptCommitter({ getConfig: getRuntimeConfig, @@ -359,6 +387,11 @@ export async function createGatewayWorkerEnvironmentRuntime(params: { githubPublication: { requestForClaim: (request) => githubPublication.requestForClaim(request), }, + portals: { + getService: () => params.getPortalRuntime()?.portalService, + carrier: workerNodePortalCarrier, + onChanged: notifyPortalChange, + }, }); const bindWorkerNodeDesktopControl = workerNodeDesktopCarrier && workerNodeDesktopStreamBroker @@ -379,7 +412,15 @@ export async function createGatewayWorkerEnvironmentRuntime(params: { bindGitHubPublication: (coordinator) => { githubPublication = coordinator; }, - bindDeviceNodeControl: deviceRuntime.bindNodeTransport, + bindDeviceNodeControl: (transport) => { + deviceRuntime.bindNodeTransport(transport); + if (workerNodeDesktopStreamBroker) { + workerNodePortalCarrier.bindRuntime({ + transport, + streamBroker: workerNodeDesktopStreamBroker, + }); + } + }, ...(bindWorkerNodeDesktopControl ? { bindWorkerNodeDesktopControl } : {}), bindNodeWorkspaceBindingResolver: (resolver) => nodeWorkerTunnelManager.bindWorkspaceBindingResolver(resolver), diff --git a/src/gateway/server.auth.default-token.suite.ts b/src/gateway/server.auth.default-token.suite.ts index ddb15bee48f4..b4acb71f27c5 100644 --- a/src/gateway/server.auth.default-token.suite.ts +++ b/src/gateway/server.auth.default-token.suite.ts @@ -168,6 +168,9 @@ export function registerDefaultAuthTokenSuite(): void { expect(payload?.features?.capabilities).toContain( GATEWAY_SERVER_CAPS.NODE_WORKER_BUNDLE_STATUS, ); + expect(payload?.features?.capabilities).toContain( + GATEWAY_SERVER_CAPS.NODE_WORKER_PORTAL_STREAM, + ); expect(payload?.features?.capabilities).toContain( GATEWAY_SERVER_CAPS.SYSTEM_AGENT_WIZARD_CANCEL, ); diff --git a/src/gateway/server/ws-connection/connect-hello.ts b/src/gateway/server/ws-connection/connect-hello.ts index 0b5d113c8ece..bdf0860ab35e 100644 --- a/src/gateway/server/ws-connection/connect-hello.ts +++ b/src/gateway/server/ws-connection/connect-hello.ts @@ -140,6 +140,7 @@ export async function sendGatewayHello( GATEWAY_SERVER_CAPS.GATEWAY_RESTART_TARGET_SAFE, GATEWAY_SERVER_CAPS.NODE_WORKER_BUNDLE_RETENTION, GATEWAY_SERVER_CAPS.NODE_WORKER_BUNDLE_STATUS, + GATEWAY_SERVER_CAPS.NODE_WORKER_PORTAL_STREAM, GATEWAY_SERVER_CAPS.SESSION_UNREAD_ACK_CONTRACT, GATEWAY_SERVER_CAPS.SYSTEM_AGENT_WIZARD_CANCEL, GATEWAY_SERVER_CAPS.SYSTEM_AGENT_SETUP_MODEL_REF, diff --git a/src/gateway/server/ws-connection/message-handler.worker.test.ts b/src/gateway/server/ws-connection/message-handler.worker.test.ts index 3edac2e2b3d1..ddae37c9510c 100644 --- a/src/gateway/server/ws-connection/message-handler.worker.test.ts +++ b/src/gateway/server/ws-connection/message-handler.worker.test.ts @@ -14,6 +14,7 @@ import { type WorkerLiveEventErrorDetails, WORKER_GITHUB_PUBLICATION_PROTOCOL_FEATURE, WORKER_LIVE_EVENT_PROTOCOL_FEATURE, + WORKER_PORTAL_PROTOCOL_FEATURE, WORKER_SESSION_TOOLS_PROTOCOL_FEATURE, type WorkerSessionToolResult, type WorkerTranscriptCommitErrorReason, @@ -61,6 +62,7 @@ const HANDSHAKE = { WORKER_LIVE_EVENT_PROTOCOL_FEATURE, WORKER_SESSION_TOOLS_PROTOCOL_FEATURE, WORKER_GITHUB_PUBLICATION_PROTOCOL_FEATURE, + WORKER_PORTAL_PROTOCOL_FEATURE, WORKER_INFERENCE_PROTOCOL_FEATURE, ], }; @@ -173,6 +175,12 @@ const SESSION_TOOL_CASES = [ toolName: "github_publish", request: { toolCallId: "call-publish", title: "Publish the fix" }, }, + { + name: "portal", + method: "worker.portal", + toolName: "portal", + request: { toolCallId: "call-portal", action: "open", port: 3000 }, + }, ] as const; const cleanups: Array<() => void> = []; @@ -656,9 +664,11 @@ describe("dedicated worker websocket protocol", () => { it.each(SESSION_TOOL_CASES)("feature-gates $name independently", async (testCase) => { const requiredFeature = - testCase.toolName === "github_publish" - ? WORKER_GITHUB_PUBLICATION_PROTOCOL_FEATURE - : WORKER_SESSION_TOOLS_PROTOCOL_FEATURE; + testCase.toolName === "portal" + ? WORKER_PORTAL_PROTOCOL_FEATURE + : testCase.toolName === "github_publish" + ? WORKER_GITHUB_PUBLICATION_PROTOCOL_FEATURE + : WORKER_SESSION_TOOLS_PROTOCOL_FEATURE; const harness = attachHarness({ identity: { ...ATTACHED_IDENTITY, @@ -676,6 +686,26 @@ describe("dedicated worker websocket protocol", () => { expect(harness.service.executeSessionTool).not.toHaveBeenCalled(); }); + it.each([ + ["an invalid action", { toolCallId: "call-portal", action: "delete", port: 3000 }], + [ + "an unexpected field", + { toolCallId: "call-portal", action: "open", port: 3000, unexpected: true }, + ], + ])("rejects portal parameters with %s before execution", async (_reason, request) => { + const harness = attachHarness({ identity: ATTACHED_IDENTITY }); + await admit(harness); + harness.sendRequest("worker.portal", request); + + await waitForWorkerProtocol(() => expect(harness.responses).toHaveLength(2)); + expect(harness.responses[1]).toMatchObject({ + ok: false, + error: { details: { reason: "invalid-frame" } }, + }); + expect(harness.service.executeSessionTool).not.toHaveBeenCalled(); + expect(harness.close).not.toHaveBeenCalled(); + }); + it("dispatches semantic transcript commits on the closed worker allowlist", async () => { const harness = attachHarness(); await admit(harness); diff --git a/src/gateway/server/ws-connection/worker-connection-dispatch.ts b/src/gateway/server/ws-connection/worker-connection-dispatch.ts new file mode 100644 index 000000000000..c18f992712cd --- /dev/null +++ b/src/gateway/server/ws-connection/worker-connection-dispatch.ts @@ -0,0 +1,329 @@ +import { + type RequestFrame, + type WorkerConnectParams, + type WorkerErrorShape, + type WorkerGitHubPublishParams, + type WorkerHeartbeatResult, + type WorkerLiveEventErrorDetails, + type WorkerLiveEventErrorShape, + type WorkerLiveEventParams, + type WorkerLiveEventResult, + type WorkerPortalParams, + type WorkerProtocolCloseReason, + type WorkerSessionsSendParams, + type WorkerSessionsSpawnParams, + type WorkerSessionToolResult, + type WorkerTranscriptCommitErrorReason, + type WorkerTranscriptCommitErrorShape, + type WorkerTranscriptCommitParams, + type WorkerTranscriptCommitResult, + WORKER_GITHUB_PUBLICATION_PROTOCOL_FEATURE, + WORKER_LIVE_EVENT_PROTOCOL_FEATURE, + WORKER_PORTAL_PROTOCOL_FEATURE, + WORKER_PROTOCOL_METHODS, + WORKER_SESSION_TOOLS_PROTOCOL_FEATURE, + WORKER_TRANSCRIPT_COMMIT_PROTOCOL_FEATURE, + validateWorkerGitHubPublishParams, + validateWorkerHeartbeatParams, + validateWorkerLiveEventParams, + validateWorkerPortalParams, + validateWorkerSessionsSendParams, + validateWorkerSessionsSpawnParams, + validateWorkerTranscriptCommitParams, +} from "../../../../packages/gateway-protocol/src/index.js"; +import { + type WorkerInferenceCancelParams, + type WorkerInferenceCancelResult, + type WorkerInferenceErrorReason, + type WorkerInferenceErrorShape, + type WorkerInferenceEventFrame, + type WorkerInferenceStartParams, + type WorkerInferenceStartResult, + type WorkerInferenceTerminalFrame, + WORKER_INFERENCE_METHODS, + WORKER_INFERENCE_PROTOCOL_FEATURE, + validateWorkerInferenceCancelParams, + validateWorkerInferenceStartParams, +} from "../../../../packages/gateway-protocol/src/schema/worker-inference.js"; +import type { WorkerConnectionIdentity } from "../../worker-environments/connection-identity.js"; +import { + workerInferenceError, + workerLiveEventError, + workerProtocolError, + workerTranscriptCommitError, +} from "./worker-connection-frames.js"; + +type WorkerServiceResult = + | { ok: true; result: TResult } + | ({ ok: false } & (TFailure | { closeReason: WorkerProtocolCloseReason })); + +export type WorkerConnectionService = { + admitWorker: ( + admission: WorkerConnectParams["admission"], + ) => Promise< + | { ok: true; identity: WorkerConnectionIdentity } + | { ok: false; reason: WorkerProtocolCloseReason } + >; + commitTranscript: ( + identity: WorkerConnectionIdentity, + request: WorkerTranscriptCommitParams, + ) => Promise< + WorkerServiceResult + >; + pushLiveEvent: ( + identity: WorkerConnectionIdentity, + request: WorkerLiveEventParams, + ) => Promise< + WorkerServiceResult + >; + validateWorkerConnection: ( + identity: WorkerConnectionIdentity, + ) => WorkerProtocolCloseReason | null; + executeSessionTool?: ( + identity: WorkerConnectionIdentity, + toolName: "sessions_spawn" | "sessions_send" | "github_publish" | "portal", + request: + | WorkerSessionsSpawnParams + | WorkerSessionsSendParams + | WorkerGitHubPublishParams + | WorkerPortalParams, + signal?: AbortSignal, + ) => Promise>; +}; + +type WorkerInferenceConnectionService = WorkerConnectionService & { + startInference?: ( + identity: WorkerConnectionIdentity, + request: WorkerInferenceStartParams, + sink: WorkerInferenceSink, + ) => + | { ok: true; result: WorkerInferenceStartResult; launch: () => void } + | { ok: false; reason: WorkerInferenceErrorReason } + | { ok: false; closeReason: WorkerProtocolCloseReason }; + cancelInference?: ( + identity: WorkerConnectionIdentity, + request: WorkerInferenceCancelParams, + ) => WorkerServiceResult; +}; + +type WorkerInferenceSink = { + connectionId: string; + send(frame: WorkerInferenceEventFrame | WorkerInferenceTerminalFrame): void; +}; + +type WorkerRespond = ( + ok: boolean, + payload?: unknown, + error?: + | WorkerErrorShape + | WorkerInferenceErrorShape + | WorkerLiveEventErrorShape + | WorkerTranscriptCommitErrorShape, +) => void; + +function rejectWorkerRequest(params: { + reason: WorkerProtocolCloseReason; + respond: WorkerRespond; + close(code: number, reason: WorkerProtocolCloseReason): void; + warn(message: string): void; +}): void { + params.warn(`worker protocol request rejected reason=${params.reason}`); + params.respond(false, undefined, workerProtocolError(params.reason)); + queueMicrotask(() => params.close(1008, params.reason)); +} + +/** Closed worker dispatcher. It never calls the generic gateway method registry. */ +export async function dispatchWorkerRequest(params: { + request: RequestFrame; + identity: WorkerConnectionIdentity; + connectionId: string; + service: WorkerInferenceConnectionService | undefined; + send(frame: unknown): void; + respond: WorkerRespond; + close(code: number, reason: WorkerProtocolCloseReason): void; + warn(message: string): void; + signal?: AbortSignal; +}): Promise { + const service = params.service; + if (!service) { + rejectWorkerRequest({ ...params, reason: "environment-unavailable" }); + return; + } + const ownershipFailure = service.validateWorkerConnection(params.identity); + if (ownershipFailure) { + rejectWorkerRequest({ ...params, reason: ownershipFailure }); + return; + } + if (params.request.method === WORKER_INFERENCE_METHODS[0]) { + if (!params.identity.protocolFeatures.includes(WORKER_INFERENCE_PROTOCOL_FEATURE)) { + rejectWorkerRequest({ ...params, reason: "method-not-allowed" }); + return; + } + if (!validateWorkerInferenceStartParams(params.request.params)) { + params.respond(false, undefined, workerInferenceError("invalid-context")); + return; + } + if (!service.startInference) { + rejectWorkerRequest({ ...params, reason: "method-not-allowed" }); + return; + } + const outcome = service.startInference(params.identity, params.request.params, { + connectionId: params.connectionId, + send: (frame) => params.send(frame), + }); + if (outcome.ok) { + // Reply before a synchronous provider can emit. + params.respond(true, outcome.result); + outcome.launch(); + return; + } + if ("closeReason" in outcome) { + rejectWorkerRequest({ ...params, reason: outcome.closeReason }); + return; + } + params.respond(false, undefined, workerInferenceError(outcome.reason)); + return; + } + if (params.request.method === WORKER_INFERENCE_METHODS[1]) { + if (!params.identity.protocolFeatures.includes(WORKER_INFERENCE_PROTOCOL_FEATURE)) { + rejectWorkerRequest({ ...params, reason: "method-not-allowed" }); + return; + } + if (!validateWorkerInferenceCancelParams(params.request.params)) { + params.respond(false, undefined, workerInferenceError("invalid-context")); + return; + } + if (!service.cancelInference) { + rejectWorkerRequest({ ...params, reason: "method-not-allowed" }); + return; + } + const outcome = service.cancelInference(params.identity, params.request.params); + if (outcome.ok) { + params.respond(true, outcome.result); + return; + } + if ("closeReason" in outcome) { + rejectWorkerRequest({ ...params, reason: outcome.closeReason }); + return; + } + params.respond(false, undefined, workerInferenceError(outcome.reason)); + return; + } + if (params.request.method === WORKER_PROTOCOL_METHODS[1]) { + if (!params.identity.protocolFeatures.includes(WORKER_TRANSCRIPT_COMMIT_PROTOCOL_FEATURE)) { + rejectWorkerRequest({ ...params, reason: "method-not-allowed" }); + return; + } + if (!validateWorkerTranscriptCommitParams(params.request.params)) { + params.respond(false, undefined, workerTranscriptCommitError("invalid-batch")); + return; + } + const outcome = await service.commitTranscript(params.identity, params.request.params); + if (outcome.ok) { + params.respond(true, outcome.result); + return; + } + if ("closeReason" in outcome) { + rejectWorkerRequest({ ...params, reason: outcome.closeReason }); + return; + } + params.respond(false, undefined, workerTranscriptCommitError(outcome.reason)); + return; + } + if (params.request.method === WORKER_PROTOCOL_METHODS[2]) { + if (!params.identity.protocolFeatures.includes(WORKER_LIVE_EVENT_PROTOCOL_FEATURE)) { + rejectWorkerRequest({ ...params, reason: "method-not-allowed" }); + return; + } + if (!validateWorkerLiveEventParams(params.request.params)) { + params.respond(false, undefined, workerLiveEventError({ reason: "invalid-event" })); + return; + } + const outcome = await service.pushLiveEvent(params.identity, params.request.params); + if (outcome.ok) { + params.respond(true, outcome.result); + return; + } + if ("closeReason" in outcome) { + rejectWorkerRequest({ ...params, reason: outcome.closeReason }); + return; + } + params.respond(false, undefined, workerLiveEventError(outcome.details)); + return; + } + if ( + params.request.method === WORKER_PROTOCOL_METHODS[3] || + params.request.method === WORKER_PROTOCOL_METHODS[4] || + params.request.method === WORKER_PROTOCOL_METHODS[5] || + params.request.method === WORKER_PROTOCOL_METHODS[6] + ) { + const isGitHubPublish = params.request.method === WORKER_PROTOCOL_METHODS[5]; + const isPortal = params.request.method === WORKER_PROTOCOL_METHODS[6]; + const requiredFeature = isPortal + ? WORKER_PORTAL_PROTOCOL_FEATURE + : isGitHubPublish + ? WORKER_GITHUB_PUBLICATION_PROTOCOL_FEATURE + : WORKER_SESSION_TOOLS_PROTOCOL_FEATURE; + if (!params.identity.protocolFeatures.includes(requiredFeature)) { + rejectWorkerRequest({ ...params, reason: "method-not-allowed" }); + return; + } + if (!service.executeSessionTool) { + rejectWorkerRequest({ ...params, reason: "method-not-allowed" }); + return; + } + const isSpawn = params.request.method === WORKER_PROTOCOL_METHODS[3]; + const requestValid = isPortal + ? validateWorkerPortalParams(params.request.params) + : isSpawn + ? validateWorkerSessionsSpawnParams(params.request.params) + : isGitHubPublish + ? validateWorkerGitHubPublishParams(params.request.params) + : validateWorkerSessionsSendParams(params.request.params); + if (!requestValid) { + params.respond(false, undefined, workerProtocolError("invalid-frame")); + return; + } + const outcome = await service.executeSessionTool( + params.identity, + isPortal + ? "portal" + : isSpawn + ? "sessions_spawn" + : isGitHubPublish + ? "github_publish" + : "sessions_send", + // SAFETY: The selected tool's matching protocol validator accepted these request params. + params.request.params as + | WorkerSessionsSpawnParams + | WorkerSessionsSendParams + | WorkerGitHubPublishParams + | WorkerPortalParams, + params.signal, + ); + if (outcome.ok) { + params.respond(true, outcome.result); + return; + } + if ("closeReason" in outcome) { + rejectWorkerRequest({ ...params, reason: outcome.closeReason }); + return; + } + params.respond(false, undefined, workerProtocolError(outcome.reason)); + return; + } + if (params.request.method !== WORKER_PROTOCOL_METHODS[0]) { + rejectWorkerRequest({ ...params, reason: "method-not-allowed" }); + return; + } + if (!validateWorkerHeartbeatParams(params.request.params)) { + rejectWorkerRequest({ ...params, reason: "invalid-heartbeat" }); + return; + } + const result: WorkerHeartbeatResult = { + receivedAtMs: Date.now(), + status: "ok", + ownerEpoch: params.identity.ownerEpoch, + }; + params.respond(true, result); +} diff --git a/src/gateway/server/ws-connection/worker-connection.ts b/src/gateway/server/ws-connection/worker-connection.ts index 6379993a7ce6..a3b8301b6bfa 100644 --- a/src/gateway/server/ws-connection/worker-connection.ts +++ b/src/gateway/server/ws-connection/worker-connection.ts @@ -3,54 +3,17 @@ import type { RawData, WebSocket } from "ws"; import { ErrorCodes, PROTOCOL_VERSION, - type RequestFrame, type WorkerConnectParams, - type WorkerGitHubPublishParams, type WorkerErrorShape, - type WorkerHeartbeatResult, - type WorkerLiveEventErrorDetails, - type WorkerLiveEventErrorShape, - type WorkerLiveEventParams, - type WorkerLiveEventResult, type WorkerProtocolCloseReason, - type WorkerSessionsSendParams, - type WorkerSessionsSpawnParams, - type WorkerSessionToolResult, - type WorkerTranscriptCommitErrorReason, - type WorkerTranscriptCommitErrorShape, - type WorkerTranscriptCommitParams, - type WorkerTranscriptCommitResult, - WORKER_LIVE_EVENT_PROTOCOL_FEATURE, - WORKER_GITHUB_PUBLICATION_PROTOCOL_FEATURE, - WORKER_SESSION_TOOLS_PROTOCOL_FEATURE, WORKER_PROTOCOL_MAX_FRAME_ID_LENGTH, WORKER_PROTOCOL_MAX_METHOD_LENGTH, WORKER_PROTOCOL_MAX_PAYLOAD_BYTES, WORKER_PROTOCOL_METHODS, - WORKER_TRANSCRIPT_COMMIT_PROTOCOL_FEATURE, validateRequestFrame, validateWorkerConnectRequestFrame, - validateWorkerHeartbeatParams, - validateWorkerLiveEventParams, - validateWorkerGitHubPublishParams, - validateWorkerSessionsSendParams, - validateWorkerSessionsSpawnParams, - validateWorkerTranscriptCommitParams, } from "../../../../packages/gateway-protocol/src/index.js"; -import { - type WorkerInferenceCancelParams, - type WorkerInferenceCancelResult, - type WorkerInferenceErrorReason, - type WorkerInferenceErrorShape, - type WorkerInferenceEventFrame, - type WorkerInferenceStartParams, - type WorkerInferenceStartResult, - type WorkerInferenceTerminalFrame, - WORKER_INFERENCE_METHODS, - WORKER_INFERENCE_PROTOCOL_FEATURE, - validateWorkerInferenceCancelParams, - validateWorkerInferenceStartParams, -} from "../../../../packages/gateway-protocol/src/schema/worker-inference.js"; +import { WORKER_INFERENCE_METHODS } from "../../../../packages/gateway-protocol/src/schema/worker-inference.js"; import { GATEWAY_STARTUP_RETRY_AFTER_MS } from "../../../../packages/gateway-protocol/src/startup-unavailable.js"; import { rawDataByteLength } from "../../../infra/ws.js"; import { @@ -60,84 +23,23 @@ import { tryBeginGatewayRootWorkAdmission, } from "../../../process/gateway-work-admission.js"; import { AUTH_RATE_LIMIT_SCOPE_WORKER_ADMISSION } from "../../auth-rate-limit.js"; -import type { WorkerConnectionIdentity } from "../../worker-environments/connection-identity.js"; import { MAX_RUNNING_WORKER_SESSION_TOOL_OPERATIONS } from "../../worker-environments/placement-session-tool-operations.js"; import { runWorkerTurnAdmissionContinuation } from "../../worker-environments/placement-turn-claim-events.js"; import type { PublicWorkerIngressContext } from "../public-worker-ingress-context.js"; import type { GatewayWsClient, WsHandshakePhase } from "../ws-types.js"; import { runWorkerAdmissionBoundary } from "./worker-admission-boundary.js"; +import { + dispatchWorkerRequest, + type WorkerConnectionService, +} from "./worker-connection-dispatch.js"; import { buildWorkerHello, - workerInferenceError, - workerLiveEventError, workerMaxPayload, workerProtocolError, - workerTranscriptCommitError, } from "./worker-connection-frames.js"; -type WorkerServiceResult = - | { ok: true; result: TResult } - | ({ ok: false } & (TFailure | { closeReason: WorkerProtocolCloseReason })); +export type { WorkerConnectionService } from "./worker-connection-dispatch.js"; -export type WorkerConnectionService = { - admitWorker: ( - admission: WorkerConnectParams["admission"], - ) => Promise< - | { ok: true; identity: WorkerConnectionIdentity } - | { ok: false; reason: WorkerProtocolCloseReason } - >; - commitTranscript: ( - identity: WorkerConnectionIdentity, - request: WorkerTranscriptCommitParams, - ) => Promise< - WorkerServiceResult - >; - pushLiveEvent: ( - identity: WorkerConnectionIdentity, - request: WorkerLiveEventParams, - ) => Promise< - WorkerServiceResult - >; - validateWorkerConnection: ( - identity: WorkerConnectionIdentity, - ) => WorkerProtocolCloseReason | null; - executeSessionTool?: ( - identity: WorkerConnectionIdentity, - toolName: "sessions_spawn" | "sessions_send" | "github_publish", - request: WorkerSessionsSpawnParams | WorkerSessionsSendParams | WorkerGitHubPublishParams, - signal?: AbortSignal, - ) => Promise>; -}; - -type WorkerInferenceConnectionService = WorkerConnectionService & { - startInference?: ( - identity: WorkerConnectionIdentity, - request: WorkerInferenceStartParams, - sink: WorkerInferenceSink, - ) => - | { ok: true; result: WorkerInferenceStartResult; launch: () => void } - | { ok: false; reason: WorkerInferenceErrorReason } - | { ok: false; closeReason: WorkerProtocolCloseReason }; - cancelInference?: ( - identity: WorkerConnectionIdentity, - request: WorkerInferenceCancelParams, - ) => WorkerServiceResult; -}; - -type WorkerInferenceSink = { - connectionId: string; - send(frame: WorkerInferenceEventFrame | WorkerInferenceTerminalFrame): void; -}; - -type WorkerRespond = ( - ok: boolean, - payload?: unknown, - error?: - | WorkerErrorShape - | WorkerInferenceErrorShape - | WorkerLiveEventErrorShape - | WorkerTranscriptCommitErrorShape, -) => void; type WorkerLogger = { warn(message: string): void }; const MAX_QUEUED_WORKER_FRAMES = 16; const MAX_QUEUED_WORKER_BYTES = 32 * 1024 * 1024; @@ -162,206 +64,13 @@ type WorkerWsMessageHandlerParams = { publicAdmission?: PublicWorkerIngressContext; }; -function rejectWorkerRequest(params: { - reason: WorkerProtocolCloseReason; - respond: WorkerRespond; - close(code: number, reason: WorkerProtocolCloseReason): void; - warn(message: string): void; -}): void { - params.warn(`worker protocol request rejected reason=${params.reason}`); - params.respond(false, undefined, workerProtocolError(params.reason)); - queueMicrotask(() => params.close(1008, params.reason)); -} - function setSocketMaxPayload(socket: WebSocket, maxPayload: number): void { - const receiver = (socket as { _receiver?: { _maxPayload?: number } })["_receiver"]; + const receiver = (socket as { _receiver?: unknown })["_receiver"]; if (receiver) { - receiver["_maxPayload"] = maxPayload; + (receiver as { _maxPayload?: number })["_maxPayload"] = maxPayload; } } -/** Closed worker dispatcher. It never calls the generic gateway method registry. */ -async function dispatchWorkerRequest(params: { - request: RequestFrame; - identity: WorkerConnectionIdentity; - connectionId: string; - service: WorkerInferenceConnectionService | undefined; - send(frame: unknown): void; - respond: WorkerRespond; - close(code: number, reason: WorkerProtocolCloseReason): void; - warn(message: string): void; - signal?: AbortSignal; -}): Promise { - const service = params.service; - if (!service) { - rejectWorkerRequest({ ...params, reason: "environment-unavailable" }); - return; - } - const ownershipFailure = service.validateWorkerConnection(params.identity); - if (ownershipFailure) { - rejectWorkerRequest({ ...params, reason: ownershipFailure }); - return; - } - if (params.request.method === WORKER_INFERENCE_METHODS[0]) { - if (!params.identity.protocolFeatures.includes(WORKER_INFERENCE_PROTOCOL_FEATURE)) { - rejectWorkerRequest({ ...params, reason: "method-not-allowed" }); - return; - } - if (!validateWorkerInferenceStartParams(params.request.params)) { - params.respond(false, undefined, workerInferenceError("invalid-context")); - return; - } - if (!service.startInference) { - rejectWorkerRequest({ ...params, reason: "method-not-allowed" }); - return; - } - const outcome = service.startInference(params.identity, params.request.params, { - connectionId: params.connectionId, - send: (frame) => params.send(frame), - }); - if (outcome.ok) { - // Reply before a synchronous provider can emit. - params.respond(true, outcome.result); - outcome.launch(); - return; - } - if ("closeReason" in outcome) { - rejectWorkerRequest({ ...params, reason: outcome.closeReason }); - return; - } - params.respond(false, undefined, workerInferenceError(outcome.reason)); - return; - } - if (params.request.method === WORKER_INFERENCE_METHODS[1]) { - if (!params.identity.protocolFeatures.includes(WORKER_INFERENCE_PROTOCOL_FEATURE)) { - rejectWorkerRequest({ ...params, reason: "method-not-allowed" }); - return; - } - if (!validateWorkerInferenceCancelParams(params.request.params)) { - params.respond(false, undefined, workerInferenceError("invalid-context")); - return; - } - if (!service.cancelInference) { - rejectWorkerRequest({ ...params, reason: "method-not-allowed" }); - return; - } - const outcome = service.cancelInference(params.identity, params.request.params); - if (outcome.ok) { - params.respond(true, outcome.result); - return; - } - if ("closeReason" in outcome) { - rejectWorkerRequest({ ...params, reason: outcome.closeReason }); - return; - } - params.respond(false, undefined, workerInferenceError(outcome.reason)); - return; - } - if (params.request.method === WORKER_PROTOCOL_METHODS[1]) { - if (!params.identity.protocolFeatures.includes(WORKER_TRANSCRIPT_COMMIT_PROTOCOL_FEATURE)) { - rejectWorkerRequest({ ...params, reason: "method-not-allowed" }); - return; - } - if (!validateWorkerTranscriptCommitParams(params.request.params)) { - params.respond(false, undefined, workerTranscriptCommitError("invalid-batch")); - return; - } - const outcome = await service.commitTranscript(params.identity, params.request.params); - if (outcome.ok) { - params.respond(true, outcome.result); - return; - } - if ("closeReason" in outcome) { - rejectWorkerRequest({ ...params, reason: outcome.closeReason }); - return; - } - params.respond(false, undefined, workerTranscriptCommitError(outcome.reason)); - return; - } - if (params.request.method === WORKER_PROTOCOL_METHODS[2]) { - if (!params.identity.protocolFeatures.includes(WORKER_LIVE_EVENT_PROTOCOL_FEATURE)) { - rejectWorkerRequest({ ...params, reason: "method-not-allowed" }); - return; - } - if (!validateWorkerLiveEventParams(params.request.params)) { - params.respond(false, undefined, workerLiveEventError({ reason: "invalid-event" })); - return; - } - const outcome = await service.pushLiveEvent(params.identity, params.request.params); - if (outcome.ok) { - params.respond(true, outcome.result); - return; - } - if ("closeReason" in outcome) { - rejectWorkerRequest({ ...params, reason: outcome.closeReason }); - return; - } - params.respond(false, undefined, workerLiveEventError(outcome.details)); - return; - } - if ( - params.request.method === WORKER_PROTOCOL_METHODS[3] || - params.request.method === WORKER_PROTOCOL_METHODS[4] || - params.request.method === WORKER_PROTOCOL_METHODS[5] - ) { - const isGitHubPublish = params.request.method === WORKER_PROTOCOL_METHODS[5]; - const requiredFeature = isGitHubPublish - ? WORKER_GITHUB_PUBLICATION_PROTOCOL_FEATURE - : WORKER_SESSION_TOOLS_PROTOCOL_FEATURE; - if (!params.identity.protocolFeatures.includes(requiredFeature)) { - rejectWorkerRequest({ ...params, reason: "method-not-allowed" }); - return; - } - if (!service.executeSessionTool) { - rejectWorkerRequest({ ...params, reason: "method-not-allowed" }); - return; - } - const isSpawn = params.request.method === WORKER_PROTOCOL_METHODS[3]; - const requestValid = isSpawn - ? validateWorkerSessionsSpawnParams(params.request.params) - : isGitHubPublish - ? validateWorkerGitHubPublishParams(params.request.params) - : validateWorkerSessionsSendParams(params.request.params); - if (!requestValid) { - params.respond(false, undefined, workerProtocolError("invalid-frame")); - return; - } - const outcome = await service.executeSessionTool( - params.identity, - isSpawn ? "sessions_spawn" : isGitHubPublish ? "github_publish" : "sessions_send", - params.request.params as - | WorkerSessionsSpawnParams - | WorkerSessionsSendParams - | WorkerGitHubPublishParams, - params.signal, - ); - if (outcome.ok) { - params.respond(true, outcome.result); - return; - } - if ("closeReason" in outcome) { - rejectWorkerRequest({ ...params, reason: outcome.closeReason }); - return; - } - params.respond(false, undefined, workerProtocolError(outcome.reason)); - return; - } - if (params.request.method !== WORKER_PROTOCOL_METHODS[0]) { - rejectWorkerRequest({ ...params, reason: "method-not-allowed" }); - return; - } - if (!validateWorkerHeartbeatParams(params.request.params)) { - rejectWorkerRequest({ ...params, reason: "invalid-heartbeat" }); - return; - } - const result: WorkerHeartbeatResult = { - receivedAtMs: Date.now(), - status: "ok", - ownerEpoch: params.identity.ownerEpoch, - }; - params.respond(true, result); -} - /** Dedicated ingress handler: worker frames never enter the generic message handler. */ export function attachWorkerWsMessageHandler(params: WorkerWsMessageHandlerParams): () => void { let expiryTimer: ReturnType | undefined; @@ -582,6 +291,7 @@ export function attachWorkerWsMessageHandler(params: WorkerWsMessageHandlerParam parsed.method === WORKER_PROTOCOL_METHODS[3] || parsed.method === WORKER_PROTOCOL_METHODS[4] || parsed.method === WORKER_PROTOCOL_METHODS[5] || + parsed.method === WORKER_PROTOCOL_METHODS[6] || parsed.method === WORKER_INFERENCE_METHODS[0] || parsed.method === WORKER_INFERENCE_METHODS[1] ) { @@ -591,7 +301,11 @@ export function attachWorkerWsMessageHandler(params: WorkerWsMessageHandlerParam closeWorker(1008, "environment-unavailable"); return; } - const respond = (ok: boolean, payload?: unknown, error?: Parameters[2]) => { + const respond = ( + ok: boolean, + payload?: unknown, + error?: Parameters[0]["respond"]>[2], + ) => { if (disposed || params.isClosed() || params.getClient() !== client || client.invalidated) { return; } @@ -616,7 +330,8 @@ export function attachWorkerWsMessageHandler(params: WorkerWsMessageHandlerParam const isLongSessionOperation = parsed.method === WORKER_PROTOCOL_METHODS[3] || parsed.method === WORKER_PROTOCOL_METHODS[4] || - parsed.method === WORKER_PROTOCOL_METHODS[5]; + parsed.method === WORKER_PROTOCOL_METHODS[5] || + parsed.method === WORKER_PROTOCOL_METHODS[6]; if (isLongSessionOperation) { if (sessionOperations.has(parsed.id)) { failFrame(1008, "invalid-frame"); diff --git a/src/gateway/worker-environments/portal-node-carrier.test.ts b/src/gateway/worker-environments/portal-node-carrier.test.ts new file mode 100644 index 000000000000..01e5d2a1f7c3 --- /dev/null +++ b/src/gateway/worker-environments/portal-node-carrier.test.ts @@ -0,0 +1,293 @@ +import { PassThrough } from "node:stream"; +import { describe, expect, it, vi } from "vitest"; +import { GATEWAY_CLIENT_IDS } from "../../../packages/gateway-protocol/src/client-info.js"; +import { NODE_WORKER_PORTAL_STREAM_COMMAND } from "../../infra/node-commands.js"; +import { + NODE_WORKER_PORTAL_STREAM_VERSION, + NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE, +} from "../../infra/node-runner-inventory.js"; +import type { NodeDesktopStreamBroker } from "../desktop/node-stream-broker.js"; +import type { + NodeWorkerSupervisorNodeProof, + NodeWorkerSupervisorTransport, +} from "../node-registry-private.js"; +import { createWorkerNodePortalCarrier } from "./portal-node-carrier.js"; +import * as support from "./service.test-support.js"; +import type { WorkerEnvironmentRecord } from "./store.js"; + +function deferredPortalValue() { + let resolve!: (value: T) => void; + let reject!: (error: Error) => void; + const promise = new Promise((promiseResolve, promiseReject) => { + resolve = promiseResolve; + reject = promiseReject; + }); + void promise.catch(() => undefined); + return { promise, reject, resolve }; +} + +function portalNodeProof(nodeId: string): NodeWorkerSupervisorNodeProof { + return { + nodeId, + connId: "conn-1", + pairingIdentity: "identity-1", + pairingGeneration: "generation-1", + clientId: GATEWAY_CLIENT_IDS.NODE_HOST, + clientMode: "node", + protocolFeature: NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE, + workerHost: { + enabled: true, + capacity: { total: 1, available: 0 }, + portalStream: NODE_WORKER_PORTAL_STREAM_VERSION, + }, + commands: [], + }; +} + +function fakePortalBroker() { + const attachments: Array>> = []; + const streams: PassThrough[] = []; + const broker = { + mintPortal: vi.fn(() => { + const attached = deferredPortalValue<{ stream: PassThrough }>(); + attachments.push(attached); + return { + ticket: "a".repeat(48), + attachPath: `/node-portal/attach?ticket=${"a".repeat(48)}`, + expiresAtMs: support.testState.nowMs + 60_000, + attached: attached.promise, + cancel: () => attached.reject(new Error("ticket cancelled")), + }; + }), + } as unknown as NodeDesktopStreamBroker; + return { + broker, + attachNext() { + const attached = attachments.shift(); + if (!attached) { + throw new Error("expected pending portal attach"); + } + const stream = new PassThrough(); + streams.push(stream); + attached.resolve({ stream }); + return stream; + }, + streams, + }; +} + +function pendingPortalTransport(params: { + proof: NodeWorkerSupervisorNodeProof; + isProofCurrent: () => boolean; +}) { + type InvokeResult = Awaited>; + const completions: Array<(result: InvokeResult) => void> = []; + const invoke = vi.fn( + async (request) => + await new Promise((resolve) => { + completions.push(resolve); + const abort = () => + resolve({ ok: false, error: { code: "ABORTED", message: "invoke aborted" } }); + if (request.signal?.aborted) { + abort(); + } else { + request.signal?.addEventListener("abort", abort, { once: true }); + } + }), + ); + const transport: NodeWorkerSupervisorTransport = { + listCurrentNodes: async () => [params.proof], + hasCurrentRunner: (nodeId) => nodeId === params.proof.nodeId && params.isProofCurrent(), + isCurrent: () => params.isProofCurrent(), + invoke, + }; + return { + invoke, + transport, + dropNext() { + const complete = completions.shift(); + if (!complete) { + throw new Error("expected active portal invocation"); + } + complete({ ok: false, error: { code: "DISCONNECTED", message: "node disconnected" } }); + }, + }; +} + +describe("worker node portal carrier", () => { + support.setupWorkerEnvironmentServiceSuite(); + + it("advertises only current node placements with the versioned portal stream capability", async () => { + const record = support.seedReadyNodeDesktop("worker-node-portal-capability"); + let current: WorkerEnvironmentRecord | undefined = record; + let proofCurrent = true; + const proof = portalNodeProof(record.nodeDeviceId!); + const transport = pendingPortalTransport({ proof, isProofCurrent: () => proofCurrent }); + const carrier = createWorkerNodePortalCarrier({ store: { get: () => current } }); + + await expect(carrier.supports(record.environmentId, record.ownerEpoch)).resolves.toBe(false); + carrier.bindRuntime({ + transport: transport.transport, + streamBroker: fakePortalBroker().broker, + }); + await expect(carrier.supports(record.environmentId, record.ownerEpoch)).resolves.toBe(true); + await expect(carrier.supports(record.environmentId, record.ownerEpoch + 1)).resolves.toBe( + false, + ); + + proof.workerHost.portalStream = undefined; + await expect(carrier.supports(record.environmentId, record.ownerEpoch)).resolves.toBe(false); + await expect( + carrier.open({ + environmentId: record.environmentId, + ownerEpoch: record.ownerEpoch, + remotePort: 4321, + }), + ).rejects.toThrow("sessions.move"); + proof.workerHost.portalStream = NODE_WORKER_PORTAL_STREAM_VERSION; + proofCurrent = false; + await expect(carrier.supports(record.environmentId, record.ownerEpoch)).resolves.toBe(false); + current = undefined; + await expect(carrier.supports(record.environmentId, record.ownerEpoch)).resolves.toBe(false); + expect(transport.invoke).not.toHaveBeenCalled(); + }); + + it("opens one ticketed node duplex per portal connection and closes its owned streams", async () => { + const record = support.seedReadyNodeDesktop("worker-node-portal-streams"); + const proof = portalNodeProof(record.nodeDeviceId!); + const transport = pendingPortalTransport({ proof, isProofCurrent: () => true }); + const streamed = fakePortalBroker(); + const carrier = createWorkerNodePortalCarrier({ store: support.testState.store }); + carrier.bindRuntime({ transport: transport.transport, streamBroker: streamed.broker }); + + const portal = await carrier.open({ + environmentId: record.environmentId, + ownerEpoch: record.ownerEpoch, + remotePort: 4321, + }); + expect(transport.invoke).not.toHaveBeenCalled(); + + const firstConnection = portal.connect(); + await support.waitForFast(() => expect(transport.invoke).toHaveBeenCalledOnce()); + const firstStream = streamed.attachNext(); + await expect(firstConnection).resolves.toBe(firstStream); + + const secondConnection = portal.connect(); + await support.waitForFast(() => expect(transport.invoke).toHaveBeenCalledTimes(2)); + const secondStream = streamed.attachNext(); + await expect(secondConnection).resolves.toBe(secondStream); + expect(transport.invoke).toHaveBeenCalledWith( + expect.objectContaining({ + command: NODE_WORKER_PORTAL_STREAM_COMMAND, + params: { + ticket: "a".repeat(48), + attachPath: `/node-portal/attach?ticket=${"a".repeat(48)}`, + port: 4321, + }, + timeoutMs: 0, + }), + ); + + await portal.close(); + expect(firstStream.destroyed).toBe(true); + expect(secondStream.destroyed).toBe(true); + await expect(portal.connect()).rejects.toThrow("sessions.move"); + }); + + it.each([ + ["lease", (record: WorkerEnvironmentRecord) => ({ ...record, leaseId: "lease:replacement" })], + [ + "node", + (record: WorkerEnvironmentRecord) => ({ ...record, nodeDeviceId: "node:replacement" }), + ], + [ + "epoch", + (record: WorkerEnvironmentRecord) => ({ ...record, ownerEpoch: record.ownerEpoch + 1 }), + ], + ["state", (record: WorkerEnvironmentRecord) => ({ ...record, state: "draining" as const })], + [ + "destroy intent", + (record: WorkerEnvironmentRecord) => ({ ...record, destroyRequestedAtMs: 2_000 }), + ], + ] as const)("rejects the attached stream when its durable %s changes", async (_name, mutate) => { + const record = support.seedReadyNodeDesktop(`worker-node-portal-stale-${_name}`); + let current: WorkerEnvironmentRecord | undefined = record; + const transport = pendingPortalTransport({ + proof: portalNodeProof(record.nodeDeviceId!), + isProofCurrent: () => true, + }); + const streamed = fakePortalBroker(); + const carrier = createWorkerNodePortalCarrier({ store: { get: () => current } }); + carrier.bindRuntime({ transport: transport.transport, streamBroker: streamed.broker }); + const portal = await carrier.open({ + environmentId: record.environmentId, + ownerEpoch: record.ownerEpoch, + remotePort: 4321, + }); + + const connection = portal.connect(); + await support.waitForFast(() => expect(transport.invoke).toHaveBeenCalledOnce()); + current = mutate(record) as WorkerEnvironmentRecord; + const stream = streamed.attachNext(); + + await expect(connection).rejects.toThrow("owner changed before attachment"); + expect(stream.destroyed).toBe(true); + await portal.close(); + }); + + it("destroys a disconnected node stream while retaining the portal for a new connection", async () => { + const record = support.seedReadyNodeDesktop("worker-node-portal-reconnect"); + const transport = pendingPortalTransport({ + proof: portalNodeProof(record.nodeDeviceId!), + isProofCurrent: () => true, + }); + const streamed = fakePortalBroker(); + const carrier = createWorkerNodePortalCarrier({ store: support.testState.store }); + carrier.bindRuntime({ transport: transport.transport, streamBroker: streamed.broker }); + const portal = await carrier.open({ + environmentId: record.environmentId, + ownerEpoch: record.ownerEpoch, + remotePort: 4321, + }); + + const firstConnection = portal.connect(); + await support.waitForFast(() => expect(transport.invoke).toHaveBeenCalledOnce()); + const firstStream = streamed.attachNext(); + await firstConnection; + transport.dropNext(); + await support.waitForFast(() => expect(firstStream.destroyed).toBe(true)); + + const recoveredConnection = portal.connect(); + await support.waitForFast(() => expect(transport.invoke).toHaveBeenCalledTimes(2)); + const recoveredStream = streamed.attachNext(); + await expect(recoveredConnection).resolves.toBe(recoveredStream); + + await carrier.stop(record.environmentId, record.ownerEpoch); + expect(recoveredStream.destroyed).toBe(true); + }); + + it("aborts an owner that is stopped while node discovery is still pending", async () => { + const record = support.seedReadyNodeDesktop("worker-node-portal-pending-discovery"); + const pendingNodes = deferredPortalValue(); + const transport = pendingPortalTransport({ + proof: portalNodeProof(record.nodeDeviceId!), + isProofCurrent: () => true, + }); + transport.transport.listCurrentNodes = () => pendingNodes.promise; + const carrier = createWorkerNodePortalCarrier({ store: support.testState.store }); + carrier.bindRuntime({ + transport: transport.transport, + streamBroker: fakePortalBroker().broker, + }); + + const opening = carrier.open({ + environmentId: record.environmentId, + ownerEpoch: record.ownerEpoch, + remotePort: 4321, + }); + await carrier.stop(record.environmentId, record.ownerEpoch); + + await expect(opening).rejects.toThrow("owner stopped"); + expect(transport.invoke).not.toHaveBeenCalled(); + }); +}); diff --git a/src/gateway/worker-environments/portal-node-carrier.ts b/src/gateway/worker-environments/portal-node-carrier.ts new file mode 100644 index 000000000000..ea0859741987 --- /dev/null +++ b/src/gateway/worker-environments/portal-node-carrier.ts @@ -0,0 +1,291 @@ +import type { Duplex } from "node:stream"; +import { NODE_WORKER_PORTAL_STREAM_COMMAND } from "../../infra/node-commands.js"; +import { NODE_WORKER_PORTAL_STREAM_VERSION } from "../../infra/node-runner-inventory.js"; +import type { NodeDesktopStreamBroker } from "../desktop/node-stream-broker.js"; +import type { + NodeWorkerSupervisorNodeProof, + NodeWorkerSupervisorTransport, +} from "../node-registry-private.js"; +import type { WorkerEnvironmentRecord, WorkerEnvironmentStore } from "./store.js"; + +type NodePortalBinding = { + environmentId: string; + leaseId: string; + nodeDeviceId: string; + ownerEpoch: number; +}; + +type NodePortalRuntime = { + transport: NodeWorkerSupervisorTransport; + streamBroker: NodeDesktopStreamBroker; +}; + +type ActiveNodePortal = { + binding: NodePortalBinding; + controller: AbortController; + streams: Set; + closed: boolean; +}; + +type ActiveNodePortalStream = { + portal: ActiveNodePortal; + controller: AbortController; + ticket?: ReturnType; + invocation?: ReturnType; + stream?: Duplex; + stopped: boolean; +}; + +const UNSUPPORTED_NODE_PORTAL_MESSAGE = + "Portals require a current cloud-worker node with portal stream support; move the session back to the gateway with sessions.move"; + +function snapshotNodePortalBinding( + record: WorkerEnvironmentRecord | undefined, + ownerEpoch: number, +): NodePortalBinding { + if ( + !record || + (record.state !== "ready" && record.state !== "idle" && record.state !== "attached") || + record.destroyRequestedAtMs !== null || + !record.leaseId || + !record.nodeDeviceId || + record.sshEndpoint !== null || + record.ownerEpoch !== ownerEpoch + ) { + throw new Error(UNSUPPORTED_NODE_PORTAL_MESSAGE); + } + return { + environmentId: record.environmentId, + leaseId: record.leaseId, + nodeDeviceId: record.nodeDeviceId, + ownerEpoch: record.ownerEpoch, + }; +} + +function isNodePortalBindingCurrent( + store: Pick, + binding: NodePortalBinding, +): boolean { + const current = store.get(binding.environmentId); + return Boolean( + current && + (current.state === "ready" || current.state === "idle" || current.state === "attached") && + current.destroyRequestedAtMs === null && + current.leaseId === binding.leaseId && + current.nodeDeviceId === binding.nodeDeviceId && + current.sshEndpoint === null && + current.ownerEpoch === binding.ownerEpoch, + ); +} + +function nodePortalAbortError(signal: AbortSignal): Error { + return signal.reason instanceof Error + ? signal.reason + : new Error("Worker environment node portal owner stopped"); +} + +function raceNodePortalAbort(operation: Promise, signal: AbortSignal): Promise { + if (signal.aborted) { + return Promise.reject(nodePortalAbortError(signal)); + } + return new Promise((resolve, reject) => { + const onAbort = () => reject(nodePortalAbortError(signal)); + signal.addEventListener("abort", onAbort, { once: true }); + void operation.then( + (value) => { + signal.removeEventListener("abort", onAbort); + resolve(value); + }, + (error: unknown) => { + signal.removeEventListener("abort", onAbort); + reject(error instanceof Error ? error : new Error(String(error))); + }, + ); + }); +} + +/** Opens one ticketed node connection per request while its durable portal owner remains current. */ +export function createWorkerNodePortalCarrier(options: { + store: Pick; +}) { + let runtime: NodePortalRuntime | undefined; + const activePortals = new Set(); + + const bindingIsCurrent = ( + binding: NodePortalBinding, + capturedRuntime: NodePortalRuntime, + node: NodeWorkerSupervisorNodeProof, + ): boolean => + runtime === capturedRuntime && + isNodePortalBindingCurrent(options.store, binding) && + node.workerHost.portalStream === NODE_WORKER_PORTAL_STREAM_VERSION && + capturedRuntime.transport.isCurrent(node, false); + + const findCurrentNode = async ( + binding: NodePortalBinding, + capturedRuntime: NodePortalRuntime, + signal?: AbortSignal, + ): Promise => { + const discovery = capturedRuntime.transport.listCurrentNodes(); + const nodes = signal ? await raceNodePortalAbort(discovery, signal) : await discovery; + signal?.throwIfAborted(); + const node = nodes.find((candidate) => candidate.nodeId === binding.nodeDeviceId); + if (!node || !bindingIsCurrent(binding, capturedRuntime, node)) { + throw new Error(UNSUPPORTED_NODE_PORTAL_MESSAGE); + } + return node; + }; + + const retireStream = (active: ActiveNodePortalStream): void => { + if (active.stopped) { + return; + } + active.stopped = true; + active.ticket?.cancel(); + active.controller.abort(new Error("Worker environment node portal stream stopped")); + active.stream?.destroy(); + active.portal.streams.delete(active); + }; + + const stopStream = async (active: ActiveNodePortalStream): Promise => { + retireStream(active); + await active.invocation?.catch(() => undefined); + }; + + const closePortal = async (portal: ActiveNodePortal): Promise => { + if (portal.closed) { + return; + } + portal.closed = true; + portal.controller.abort(new Error("Worker environment node portal owner stopped")); + activePortals.delete(portal); + await Promise.all([...portal.streams].map(stopStream)); + }; + + const connectPortal = async (portal: ActiveNodePortal, remotePort: number): Promise => { + const capturedRuntime = runtime; + if (!capturedRuntime || portal.closed || portal.controller.signal.aborted) { + throw new Error(UNSUPPORTED_NODE_PORTAL_MESSAGE); + } + const active: ActiveNodePortalStream = { + portal, + controller: new AbortController(), + stopped: false, + }; + // Publish the connection before node discovery yields so owner teardown can fence it. + portal.streams.add(active); + try { + const node = await findCurrentNode(portal.binding, capturedRuntime, active.controller.signal); + active.ticket = capturedRuntime.streamBroker.mintPortal({ + nodeId: node.nodeId, + connId: node.connId, + pairingGeneration: node.pairingGeneration, + }); + active.invocation = capturedRuntime.transport.invoke({ + node, + command: NODE_WORKER_PORTAL_STREAM_COMMAND, + params: { + ticket: active.ticket.ticket, + attachPath: active.ticket.attachPath, + port: remotePort, + }, + timeoutMs: 0, + signal: active.controller.signal, + isDispatchAuthorized: () => + !portal.closed && bindingIsCurrent(portal.binding, capturedRuntime, node), + }); + // The invocation lives for the splice; finishing before attachment is a dial failure. + const invocationFinished = active.invocation.then((result) => { + throw new Error( + result.error?.message?.trim() || + "Worker environment node portal closed before attachment", + ); + }); + void invocationFinished.catch(() => undefined); + const attached = await Promise.race([active.ticket.attached, invocationFinished]); + active.stream = attached.stream; + if (portal.closed || !bindingIsCurrent(portal.binding, capturedRuntime, node)) { + throw new Error("Worker environment node portal owner changed before attachment"); + } + active.stream.once("close", () => retireStream(active)); + void active.invocation.finally(() => retireStream(active)).catch(() => undefined); + return active.stream; + } catch (error) { + await stopStream(active); + throw error; + } + }; + + return { + bindRuntime(next: NodePortalRuntime): void { + if (runtime && runtime !== next) { + for (const portal of activePortals) { + for (const stream of portal.streams) { + retireStream(stream); + } + } + } + runtime = next; + }, + async supports(environmentId: string, ownerEpoch: number): Promise { + const capturedRuntime = runtime; + if (!capturedRuntime) { + return false; + } + try { + const binding = snapshotNodePortalBinding(options.store.get(environmentId), ownerEpoch); + await findCurrentNode(binding, capturedRuntime); + return true; + } catch { + return false; + } + }, + async open(request: { + environmentId: string; + ownerEpoch: number; + remotePort: number; + }): Promise<{ connect: () => Promise; close: () => Promise }> { + const binding = snapshotNodePortalBinding( + options.store.get(request.environmentId), + request.ownerEpoch, + ); + const capturedRuntime = runtime; + if (!capturedRuntime) { + throw new Error(UNSUPPORTED_NODE_PORTAL_MESSAGE); + } + const portal: ActiveNodePortal = { + binding, + controller: new AbortController(), + streams: new Set(), + closed: false, + }; + activePortals.add(portal); + try { + await findCurrentNode(binding, capturedRuntime, portal.controller.signal); + return { + connect: () => connectPortal(portal, request.remotePort), + close: () => closePortal(portal), + }; + } catch (error) { + await closePortal(portal); + throw error; + } + }, + async stop(environmentId: string, ownerEpoch?: number): Promise { + await Promise.all( + [...activePortals] + .filter( + (portal) => + portal.binding.environmentId === environmentId && + (ownerEpoch === undefined || portal.binding.ownerEpoch === ownerEpoch), + ) + .map(closePortal), + ); + }, + async stopAll(): Promise { + await Promise.all([...activePortals].map(closePortal)); + }, + }; +} + +export type WorkerNodePortalCarrier = ReturnType; diff --git a/src/gateway/worker-environments/service.ts b/src/gateway/worker-environments/service.ts index cebcc3e32668..2a05bda20571 100644 --- a/src/gateway/worker-environments/service.ts +++ b/src/gateway/worker-environments/service.ts @@ -1,5 +1,6 @@ import type { WorkerGitHubPublishParams, + WorkerPortalParams, WorkerSessionsSendParams, WorkerSessionsSpawnParams, WorkerSessionToolResult, @@ -28,6 +29,7 @@ import type { WorkerLiveEventReceiver } from "./live-events.js"; import type { WorkerNodeDesktopCarrier } from "./node-desktop-carrier.js"; import type { NodeWorkerTunnelManager } from "./node-worker-tunnel.js"; import type { WorkerSessionPlacementGate } from "./placement-worker-gate.js"; +import type { WorkerNodePortalCarrier } from "./portal-node-carrier.js"; import { createWorkerProviderLifecycle } from "./provider-lifecycle.js"; import type { WorkerProviderLifecycleInputOptions } from "./provider-lifecycle.types.js"; import type { WorkerEnvironmentState } from "./state.js"; @@ -67,6 +69,8 @@ type WorkerEnvironmentServiceOptions = WorkerProviderLifecycleInputOptions & { tunnelManager?: WorkerTunnelManager; nodeTunnelManager?: NodeWorkerTunnelManager; nodeDesktopCarrier?: WorkerNodeDesktopCarrier; + nodePortalCarrier?: WorkerNodePortalCarrier; + closeWorkerPortals?: (environmentId: string, ownerEpoch?: number) => Promise; stopNodeEnrollmentWaits?: () => void; stopNodeWorkerBundleTransfers?: () => void; reconcileIntervalMs?: number; @@ -108,6 +112,12 @@ type WorkerEnvironmentServiceOptions = WorkerProviderLifecycleInputOptions & { toolName: "github_publish"; request: WorkerGitHubPublishParams; signal?: AbortSignal; + } + | { + identity: WorkerConnectionIdentity; + toolName: "portal"; + request: WorkerPortalParams; + signal?: AbortSignal; }, ) => Promise; }; @@ -126,13 +136,18 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService const activeOperations = new Set>(); const now = options.now ?? Date.now; const tunnelLifecycle = - options.tunnelManager || options.nodeTunnelManager || options.nodeDesktopCarrier + options.tunnelManager || + options.nodeTunnelManager || + options.nodeDesktopCarrier || + options.nodePortalCarrier ? { stop: async (environmentId: string, ownerEpoch?: number) => { await Promise.all([ options.tunnelManager?.stop(environmentId, ownerEpoch), options.nodeTunnelManager?.stop(environmentId, ownerEpoch), options.nodeDesktopCarrier?.stop(environmentId, ownerEpoch), + options.nodePortalCarrier?.stop(environmentId, ownerEpoch), + options.closeWorkerPortals?.(environmentId, ownerEpoch), ]); }, } @@ -456,7 +471,7 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService options.liveEvents?.clear(); options.stopNodeWorkerBundleTransfers?.(); try { - await environmentAccess.stopAllTunnels(); + await Promise.all([environmentAccess.stopAllTunnels(), options.nodePortalCarrier?.stopAll()]); } finally { // Tunnel failures cannot release shutdown before admitted owner-bound operations drain. const reconciliation = reconcileInFlight; @@ -509,6 +524,8 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService return id ? options.resolveProvider(id)?.requiresNodeEnrollment === true : false; }, get: environmentAccess.get, + supportsNodePortal: async (environmentId: string, ownerEpoch: number) => + (await options.nodePortalCarrier?.supports(environmentId, ownerEpoch)) === true, hasPendingNodeEnrollmentSetup: (setupId: string, deviceId: string) => store.hasPendingNodeEnrollmentSetup(setupId, deviceId), listMachineOptions: async (profileId: string) => @@ -588,7 +605,13 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService acquireTurnCredential: credentialBroker.acquireTurnCredential, acknowledgeCredentialDelivery: credentialBroker.acknowledgeCredentialDelivery, startTunnel: environmentAccess.startTunnel, - stopTunnel: environmentAccess.stopTunnel, + stopTunnel: async (environmentId: string, ownerEpoch?: number) => { + await Promise.all([ + environmentAccess.stopTunnel(environmentId, ownerEpoch), + options.nodePortalCarrier?.stop(environmentId, ownerEpoch), + options.closeWorkerPortals?.(environmentId, ownerEpoch), + ]); + }, stopNodeEnrollmentWaits: options.stopNodeEnrollmentWaits, installReconcileEnvironmentGuard, reconcileEnvironment, diff --git a/src/gateway/worker-environments/worker-browser-launch-plan.ts b/src/gateway/worker-environments/worker-browser-launch-plan.ts index d6dd8043261b..28645b648759 100644 --- a/src/gateway/worker-environments/worker-browser-launch-plan.ts +++ b/src/gateway/worker-environments/worker-browser-launch-plan.ts @@ -11,6 +11,7 @@ export function resolveWorkerBrowserLaunchPlan(params: { modelRef: { provider: string; model: string }; turn: SessionPlacementTurnParams; githubPublicationAvailable?: boolean; + portalAvailable?: boolean; }): { browser?: WorkerBrowserLaunchDescriptor; toolAuthority: WorkerToolAuthority; @@ -28,6 +29,7 @@ export function resolveWorkerBrowserLaunchPlan(params: { modelRef: params.modelRef, turn: params.turn, githubPublicationAvailable: params.githubPublicationAvailable, + portalAvailable: params.portalAvailable, ...(browserAvailable ? { availableOptionalToolNames: ["browser"] } : {}), }); return { diff --git a/src/gateway/worker-environments/worker-portal-tool-executor.test.ts b/src/gateway/worker-environments/worker-portal-tool-executor.test.ts new file mode 100644 index 000000000000..047cbda7a5a0 --- /dev/null +++ b/src/gateway/worker-environments/worker-portal-tool-executor.test.ts @@ -0,0 +1,399 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { SessionEntry } from "../../config/sessions.js"; +import { + closeOpenClawStateDatabaseForTest, + openOpenClawStateDatabase, +} from "../../state/openclaw-state-db.js"; +import type { WorkerConnectionIdentity } from "./connection-identity.js"; +import { + createWorkerSessionPlacementStore, + type WorkerSessionPlacementStore, +} from "./placement-store.js"; +import { createWorkerPortalToolExecutor } from "./worker-portal-tool-executor.js"; + +const sessionEntries = vi.hoisted(() => new Map()); + +vi.mock("../session-utils.js", () => ({ + loadGatewaySessionEntryReadOnly: (sessionKey: string) => ({ + canonicalKey: sessionKey, + entry: structuredClone(sessionEntries.get(sessionKey)), + }), +})); + +const SOURCE = { + agentId: "main", + sessionId: "source-session", + sessionKey: "agent:main:dashboard:source", + environmentId: "source-environment", + ownerEpoch: 3, +}; + +const PORTAL = { + id: "worker-source-4321", + title: "Worker app", + port: 4321, + listenPort: 54321, + tokenQuery: "openclaw_portal=test-token", + url: "http://127.0.0.1:54321/?openclaw_portal=test-token", + publicUrl: "http://127.0.0.1:54321/", + origin: "cloud-profile", + createdAtMs: 1, +}; + +describe("worker portal tool execution", () => { + let root: string; + let placements: WorkerSessionPlacementStore; + let identity: WorkerConnectionIdentity; + let sourceClaim: ReturnType; + let sourceEnvironmentEpoch: number; + let sourceNodeDeviceId: string | null; + let sourceSshEndpoint: { host: string } | null; + let execute: ReturnType; + const portalOpen = vi.fn(); + const portalList = vi.fn(); + const portalWorkerList = vi.fn(); + const portalClose = vi.fn(); + const portalCarrierOpen = vi.fn(); + const portalCarrierConnect = vi.fn(); + const portalCarrierClose = vi.fn(); + const portalChanged = vi.fn(); + + beforeEach(async () => { + root = await fs.mkdtemp(path.join(await fs.realpath(os.tmpdir()), "openclaw-worker-portal-")); + const database = openOpenClawStateDatabase({ env: { OPENCLAW_STATE_DIR: root } }); + placements = createWorkerSessionPlacementStore({ database }); + let placement = placements.startDispatch(SOURCE); + placement = placements.transition({ + sessionId: SOURCE.sessionId, + from: "requested", + to: "provisioning", + expectedGeneration: placement.generation, + patch: { environmentId: SOURCE.environmentId }, + }); + placement = placements.transition({ + sessionId: SOURCE.sessionId, + from: "provisioning", + to: "syncing", + expectedGeneration: placement.generation, + patch: { workerBundleHash: "a".repeat(64) }, + }); + placement = placements.transition({ + sessionId: SOURCE.sessionId, + from: "syncing", + to: "starting", + expectedGeneration: placement.generation, + patch: { + workspaceBaseManifestRef: `manifest-${SOURCE.sessionId}`, + remoteWorkspaceDir: `/workspace/${SOURCE.sessionId}`, + }, + }); + placements.transition({ + sessionId: SOURCE.sessionId, + from: "starting", + to: "active", + expectedGeneration: placement.generation, + patch: { activeOwnerEpoch: SOURCE.ownerEpoch }, + }); + sourceClaim = placements.claimTurn({ + sessionId: SOURCE.sessionId, + agentId: SOURCE.agentId, + sessionKey: SOURCE.sessionKey, + claimId: "source-claim", + runId: "source-run", + owner: { + kind: "worker", + environmentId: SOURCE.environmentId, + ownerEpoch: SOURCE.ownerEpoch, + }, + }); + placements.authorizeWorkerTurnTools(sourceClaim, ["portal"]); + identity = { + environmentId: SOURCE.environmentId, + credentialHash: "credential-hash", + bundleHash: "a".repeat(64), + sessionId: SOURCE.sessionId, + runId: sourceClaim.runId, + turnClaim: sourceClaim, + ownerEpoch: SOURCE.ownerEpoch, + rpcSetVersion: 1, + protocolFeatures: ["worker-portal-v1"], + credentialExpiresAtMs: Date.now() + 60_000, + }; + sessionEntries.clear(); + sessionEntries.set(SOURCE.sessionKey, { sessionId: SOURCE.sessionId, updatedAt: Date.now() }); + portalOpen.mockReset().mockResolvedValue({ portal: PORTAL, created: true }); + portalList.mockReset().mockReturnValue([PORTAL]); + portalWorkerList.mockReset().mockReturnValue([PORTAL]); + portalClose.mockReset().mockResolvedValue(undefined); + portalCarrierConnect.mockReset(); + portalCarrierClose.mockReset().mockResolvedValue(undefined); + portalCarrierOpen.mockReset().mockResolvedValue({ + connect: portalCarrierConnect, + close: portalCarrierClose, + }); + portalChanged.mockReset(); + sourceEnvironmentEpoch = SOURCE.ownerEpoch; + sourceNodeDeviceId = "worker-node"; + sourceSshEndpoint = null; + execute = createWorkerPortalToolExecutor({ + placements, + portals: { + getService: () => + ({ + open: portalOpen, + list: portalList, + listWorkerPortals: portalWorkerList, + close: portalClose, + }) as never, + carrier: { open: portalCarrierOpen }, + onChanged: portalChanged, + }, + environments: { + get: (environmentId: string) => + environmentId === SOURCE.environmentId + ? { + state: "attached", + environmentId: SOURCE.environmentId, + ownerEpoch: sourceEnvironmentEpoch, + attachedSessionIds: [SOURCE.sessionId], + providerId: "fake", + profileId: "cloud-profile", + profileSnapshot: { install: "bundle", settings: { region: "source" } }, + nodeDeviceId: sourceNodeDeviceId, + sshEndpoint: sourceSshEndpoint, + } + : undefined, + } as never, + }); + }); + + afterEach(async () => { + closeOpenClawStateDatabaseForTest(); + await fs.rm(root, { recursive: true, force: true }); + }); + + it("opens, lists, and closes node-backed worker portals through the authorized source turn", async () => { + const opened = await execute({ + identity, + toolName: "portal", + request: { + toolCallId: "open-worker-portal", + action: "open", + port: 4321, + title: "Worker app", + path: "/app", + }, + }); + + expect(JSON.parse(opened.resultJson)).toMatchObject({ + details: { id: "worker-source-4321", origin: "cloud-profile", port: 4321 }, + content: [ + { + type: "text", + text: expect.stringContaining("PUBLIC_URL=http://127.0.0.1:54321/ and PORT=4321"), + }, + expect.any(Object), + ], + }); + expect(portalCarrierOpen).toHaveBeenCalledWith({ + environmentId: SOURCE.environmentId, + ownerEpoch: SOURCE.ownerEpoch, + remotePort: 4321, + }); + expect(portalOpen).toHaveBeenCalledWith({ + targetPort: 4321, + assertCurrent: expect.any(Function), + target: { + kind: "worker", + environmentId: SOURCE.environmentId, + ownerEpoch: SOURCE.ownerEpoch, + connect: portalCarrierConnect, + remotePort: 4321, + }, + onClose: portalCarrierClose, + origin: "cloud-profile", + title: "Worker app", + path: "/app", + }); + + const listed = await execute({ + identity, + toolName: "portal", + request: { toolCallId: "list-worker-portals", action: "list" }, + }); + expect(JSON.parse(listed.resultJson).details.portals).toHaveLength(1); + expect(portalWorkerList).toHaveBeenCalledWith(SOURCE.environmentId, SOURCE.ownerEpoch); + + const closed = await execute({ + identity, + toolName: "portal", + request: { + toolCallId: "close-worker-portal", + action: "close", + id: "worker-source-4321", + }, + }); + expect(JSON.parse(closed.resultJson).details).toEqual({ closed: true }); + expect(portalClose).toHaveBeenCalledWith("worker-source-4321", expect.any(Function)); + expect(portalChanged).toHaveBeenCalledTimes(2); + }); + + it("never exposes local or other-worker portal bearer URLs to a delegated worker", async () => { + portalList.mockReturnValue([ + ...portalWorkerList(), + { id: "local-portal", url: "http://127.0.0.1:54322/?openclaw_portal=local-secret" }, + { + id: "foreign-worker-portal", + url: "http://127.0.0.1:54323/?openclaw_portal=foreign-secret", + }, + ]); + portalWorkerList.mockClear(); + + const result = await execute({ + identity, + toolName: "portal", + request: { toolCallId: "list-owned-worker-portals", action: "list" }, + }); + + expect(JSON.parse(result.resultJson).details.portals).toEqual([ + expect.objectContaining({ id: "worker-source-4321" }), + ]); + expect(result.resultJson).not.toContain("local-secret"); + expect(result.resultJson).not.toContain("foreign-secret"); + expect(portalList).not.toHaveBeenCalled(); + expect(portalWorkerList).toHaveBeenCalledWith(SOURCE.environmentId, SOURCE.ownerEpoch); + }); + + it("rejects attempts to close gateway-host or other-worker portals", async () => { + for (const id of ["local-portal", "foreign-worker-portal"]) { + await expect( + execute({ + identity, + toolName: "portal", + request: { toolCallId: `close-${id}`, action: "close", id }, + }), + ).rejects.toThrow("Worker portal is not owned by the active environment"); + } + + expect(portalClose).not.toHaveBeenCalled(); + expect(portalWorkerList).toHaveBeenCalledTimes(2); + }); + + it("rejects stale owner epochs and replaced source placements before opening a portal", async () => { + await expect( + execute({ + identity: { ...identity, ownerEpoch: SOURCE.ownerEpoch + 1 }, + toolName: "portal", + request: { toolCallId: "stale-worker-portal", action: "open", port: 4321 }, + }), + ).rejects.toThrow("Worker source environment changed"); + + placements.releaseTurn(sourceClaim); + await expect( + execute({ + identity, + toolName: "portal", + request: { toolCallId: "replaced-worker-portal", action: "open", port: 4321 }, + }), + ).rejects.toThrow("Worker source session placement changed"); + expect(portalCarrierOpen).not.toHaveBeenCalled(); + }); + + it("closes a prepared worker portal carrier when its owner epoch changes while opening", async () => { + portalCarrierOpen.mockImplementationOnce(async () => { + sourceEnvironmentEpoch += 1; + return { + connect: portalCarrierConnect, + close: portalCarrierClose, + }; + }); + + await expect( + execute({ + identity, + toolName: "portal", + request: { toolCallId: "replaced-worker-forward", action: "open", port: 4321 }, + }), + ).rejects.toThrow("Worker source environment changed"); + expect(portalCarrierClose).toHaveBeenCalledOnce(); + expect(portalOpen).not.toHaveBeenCalled(); + }); + + it("never closes a reused portal when authority is lost after open", async () => { + // Regression: a revoked turn's duplicate open must not tear down the live + // portal a still-authorized predecessor established. + portalOpen.mockImplementationOnce(async () => { + sourceEnvironmentEpoch += 1; + return { portal: PORTAL, created: false }; + }); + + await expect( + execute({ + identity, + toolName: "portal", + request: { toolCallId: "reused-worker-portal", action: "open", port: 4321 }, + }), + ).rejects.toThrow("Worker source environment changed"); + expect(portalClose).not.toHaveBeenCalled(); + expect(portalCarrierClose).toHaveBeenCalled(); + }); + + it("closes the redundant carrier handle when an existing portal is reused", async () => { + portalOpen.mockResolvedValueOnce({ portal: PORTAL, created: false }); + + const result = await execute({ + identity, + toolName: "portal", + request: { toolCallId: "reuse-worker-portal", action: "open", port: 4321 }, + }); + expect(result.resultJson).toContain(PORTAL.id); + expect(portalCarrierClose).toHaveBeenCalledOnce(); + expect(portalClose).not.toHaveBeenCalled(); + }); + + it("closes only the portal this turn created when authority is lost after open", async () => { + portalOpen.mockImplementationOnce(async () => { + sourceEnvironmentEpoch += 1; + return { portal: PORTAL, created: true }; + }); + + await expect( + execute({ + identity, + toolName: "portal", + request: { toolCallId: "created-worker-portal", action: "open", port: 4321 }, + }), + ).rejects.toThrow("Worker source environment changed"); + expect(portalClose).toHaveBeenCalledWith(PORTAL.id); + }); + + it("rejects SSH-backed placements before preparing a node portal", async () => { + sourceNodeDeviceId = null; + sourceSshEndpoint = { host: "worker.example" }; + + await expect( + execute({ + identity, + toolName: "portal", + request: { toolCallId: "ssh-worker-portal", action: "open", port: 4321 }, + }), + ).rejects.toThrow("move the session back to the gateway with sessions.move"); + expect(portalCarrierOpen).not.toHaveBeenCalled(); + }); + + it("rejects worker portal access when the active turn was not granted portal authority", async () => { + placements.authorizeWorkerTurnTools(sourceClaim, ["sessions_send"]); + + await expect( + execute({ + identity, + toolName: "portal", + request: { toolCallId: "unauthorized-worker-portal", action: "open", port: 4321 }, + }), + ).rejects.toThrow("Worker session tool authority changed"); + expect(portalCarrierOpen).not.toHaveBeenCalled(); + }); +}); diff --git a/src/gateway/worker-environments/worker-portal-tool-executor.ts b/src/gateway/worker-environments/worker-portal-tool-executor.ts new file mode 100644 index 000000000000..77878c24f962 --- /dev/null +++ b/src/gateway/worker-environments/worker-portal-tool-executor.ts @@ -0,0 +1,154 @@ +import type { + WorkerPortalParams, + WorkerSessionToolResult, +} from "../../../packages/gateway-protocol/src/schema/worker-admission.js"; +import { formatPortalResult } from "../../agents/tools/portal-tool.js"; +import type { GatewayPortalService } from "../portals/portal-service.js"; +import type { WorkerConnectionIdentity } from "./connection-identity.js"; +import type { WorkerSessionPlacementStore } from "./placement-store.js"; +import type { WorkerNodePortalCarrier } from "./portal-node-carrier.js"; +import type { WorkerEnvironmentService } from "./service.js"; +import { serializeWorkerSessionToolResult } from "./worker-session-tool-result.js"; +import { resolveWorkerSessionToolSource } from "./worker-session-tool-topology.js"; + +export type WorkerPortalToolRequest = { + identity: WorkerConnectionIdentity; + toolName: "portal"; + request: WorkerPortalParams; + signal?: AbortSignal; +}; + +export type WorkerPortalToolExecutorDependencies = { + placements: WorkerSessionPlacementStore; + environments: Pick; + portals: { + getService: () => GatewayPortalService | undefined; + carrier: Pick; + onChanged: () => void; + }; +}; + +/** Executes worker portals only while their exact placement and turn retain authority. */ +export function createWorkerPortalToolExecutor(params: WorkerPortalToolExecutorDependencies) { + return async (request: WorkerPortalToolRequest): Promise => { + const assertPortalAuthority = () => { + const current = resolveWorkerSessionToolSource({ + identity: request.identity, + placements: params.placements, + }); + if (!params.placements.isWorkerTurnToolAuthorized(current.turnClaim, "portal")) { + throw new Error("Worker session tool authority changed"); + } + const environment = params.environments.get(request.identity.environmentId); + if ( + !environment || + environment.state !== "attached" || + environment.ownerEpoch !== request.identity.ownerEpoch || + environment.attachedSessionIds.length !== 1 || + environment.attachedSessionIds[0] !== current.sessionId + ) { + throw new Error("Worker source environment changed before portal operation"); + } + if (!environment.nodeDeviceId || environment.sshEndpoint !== null) { + throw new Error( + "Portals require a node-backed cloud-worker placement; move the session back to the gateway with sessions.move", + ); + } + return environment; + }; + const environment = assertPortalAuthority(); + const service = params.portals.getService(); + if (!service) { + throw new Error("Gateway portals are unavailable"); + } + request.signal?.throwIfAborted(); + if (request.request.action === "list") { + const result = { + portals: service.listWorkerPortals(environment.environmentId, environment.ownerEpoch), + }; + assertPortalAuthority(); + return { + resultJson: serializeWorkerSessionToolResult( + formatPortalResult({ action: "list", result }), + ), + }; + } + if (request.request.action === "close") { + const id = request.request.id; + if (!id) { + throw new Error("portal id required"); + } + const ownedPortals = service.listWorkerPortals( + environment.environmentId, + environment.ownerEpoch, + ); + if (!ownedPortals.some((portal) => portal.id === id)) { + throw new Error("Worker portal is not owned by the active environment"); + } + await service.close(id, assertPortalAuthority); + assertPortalAuthority(); + params.portals.onChanged(); + return { + resultJson: serializeWorkerSessionToolResult( + formatPortalResult({ action: "close", id, result: { closed: true } }), + ), + }; + } + const remotePort = request.request.port; + if (remotePort === undefined) { + throw new Error("portal port required"); + } + const connection = await params.portals.carrier.open({ + environmentId: environment.environmentId, + ownerEpoch: environment.ownerEpoch, + remotePort, + }); + let createdPortalId: string | undefined; + try { + // Node discovery can yield; a replaced turn must never publish its former owner's portal. + assertPortalAuthority(); + request.signal?.throwIfAborted(); + const opened = await service.open({ + targetPort: remotePort, + assertCurrent: assertPortalAuthority, + target: { + kind: "worker", + environmentId: environment.environmentId, + ownerEpoch: environment.ownerEpoch, + connect: connection.connect, + remotePort, + }, + onClose: connection.close, + origin: environment.profileId, + ...(request.request.title !== undefined ? { title: request.request.title } : {}), + ...(request.request.description !== undefined + ? { description: request.request.description } + : {}), + ...(request.request.path !== undefined ? { path: request.request.path } : {}), + }); + if (opened.created) { + createdPortalId = opened.portal.id; + } else { + // Reuse keeps the existing entry's connect/onClose; this turn's carrier handle is redundant. + await connection.close(); + } + assertPortalAuthority(); + params.portals.onChanged(); + return { + resultJson: serializeWorkerSessionToolResult( + formatPortalResult({ action: "open", result: opened.portal }), + ), + }; + } catch (error) { + // Only tear down what this turn created: closing a reused portal here would let a + // revoked turn destroy the live portal a still-authorized predecessor established. + if (createdPortalId) { + await service.close(createdPortalId); + params.portals.onChanged(); + } else { + await connection.close(); + } + throw error; + } + }; +} diff --git a/src/gateway/worker-environments/worker-session-tool-executor.test.ts b/src/gateway/worker-environments/worker-session-tool-executor.test.ts index 2abba9cab321..89738583a053 100644 --- a/src/gateway/worker-environments/worker-session-tool-executor.test.ts +++ b/src/gateway/worker-environments/worker-session-tool-executor.test.ts @@ -91,6 +91,8 @@ vi.mock("../../agents/tools/scoped-session-access.js", () => ({ vi.mock("../../agents/tools/in-process-gateway.js", () => ({ callAgentToolGatewayRequest: (request: unknown) => gatewayRequest(request), + callInProcessGatewayTool: (method: string, params: Record) => + gatewayRequest({ method, params }), callInProcessGatewayToolWithCreation: ( method: string, params: Record, @@ -247,6 +249,11 @@ describe("worker session tool topology", () => { placements, dispatchChild, githubPublication: { requestForClaim: githubPublicationRequest }, + portals: { + getService: () => undefined, + carrier: { open: vi.fn() }, + onChanged: vi.fn(), + }, environments: { get: (environmentId: string) => { if (environmentId === SOURCE.environmentId) { diff --git a/src/gateway/worker-environments/worker-session-tool-executor.ts b/src/gateway/worker-environments/worker-session-tool-executor.ts index 52e8c42cc5bd..2d8c6fcca937 100644 --- a/src/gateway/worker-environments/worker-session-tool-executor.ts +++ b/src/gateway/worker-environments/worker-session-tool-executor.ts @@ -35,6 +35,11 @@ import { } from "./placement-turn-claim-events.js"; import type { WorkerPlacementDispatchContract } from "./service-contract.js"; import type { WorkerEnvironmentService } from "./service.js"; +import { + createWorkerPortalToolExecutor, + type WorkerPortalToolExecutorDependencies, + type WorkerPortalToolRequest, +} from "./worker-portal-tool-executor.js"; import { serializeWorkerSessionToolResult as serializeResult, workerSessionToolErrorResult as errorResult, @@ -49,24 +54,12 @@ import { } from "./worker-session-tool-topology.js"; type WorkerSessionToolRequest = - | { - identity: WorkerConnectionIdentity; - toolName: "sessions_spawn"; - request: WorkerSessionsSpawnParams; - signal?: AbortSignal; - } - | { - identity: WorkerConnectionIdentity; - toolName: "sessions_send"; - request: WorkerSessionsSendParams; - signal?: AbortSignal; - } - | { - identity: WorkerConnectionIdentity; - toolName: "github_publish"; - request: WorkerGitHubPublishParams; - signal?: AbortSignal; - }; + | WorkerPortalToolRequest + | ({ identity: WorkerConnectionIdentity; signal?: AbortSignal } & ( + | { toolName: "sessions_spawn"; request: WorkerSessionsSpawnParams } + | { toolName: "sessions_send"; request: WorkerSessionsSendParams } + | { toolName: "github_publish"; request: WorkerGitHubPublishParams } + )); class WorkerSessionToolOutcomeUnknownError extends Error { constructor(cause: unknown) { @@ -100,8 +93,10 @@ export function createWorkerSessionToolExecutor(params: { environments: Pick; dispatchChild: WorkerPlacementDispatchContract["dispatch"]; githubPublication: Pick; + portals: WorkerPortalToolExecutorDependencies["portals"]; }) { const inFlight = new Map>(); + const executePortal = createWorkerPortalToolExecutor(params); const spawn = async (operation: { source: ExactSource; @@ -522,6 +517,9 @@ export function createWorkerSessionToolExecutor(params: { return async (request: WorkerSessionToolRequest): Promise => { const source = exactSource({ identity: request.identity, placements: params.placements }); + if (request.toolName === "portal") { + return await executePortal(request); + } if (request.toolName === "github_publish") { const assertPublicationAuthority = () => { const current = exactSource({ identity: request.identity, placements: params.placements }); diff --git a/src/gateway/worker-environments/worker-tool-authority.test.ts b/src/gateway/worker-environments/worker-tool-authority.test.ts index 65b288cf9f8b..d574456d3cc3 100644 --- a/src/gateway/worker-environments/worker-tool-authority.test.ts +++ b/src/gateway/worker-environments/worker-tool-authority.test.ts @@ -21,11 +21,13 @@ function turn(overrides: Partial = {}): SessionPlace function authority( overrides: Partial = {}, githubPublicationAvailable = false, + portalAvailable = false, ) { return resolveWorkerToolAuthority({ modelRef: { provider: "openai", model: "gpt-test" }, turn: turn(overrides), githubPublicationAvailable, + portalAvailable, }).allowedToolNames; } @@ -76,6 +78,8 @@ describe("resolveWorkerToolAuthority", () => { expect(authority({ toolsAllow: [] })).toEqual([]); expect(authority({ toolsAllow: ["web_search"] })).toEqual([]); expect(authority({ toolsAllow: ["sessions_send"] })).toEqual(["sessions_send"]); + expect(authority({ toolsAllow: ["portal"] })).toEqual([]); + expect(authority({ toolsAllow: ["portal"] }, false, true)).toEqual(["portal"]); expect(authority({ toolsAllow: ["github_publish"] })).toEqual([]); expect(authority({ toolsAllow: ["github_publish"] }, true)).toEqual(["github_publish"]); }); @@ -85,6 +89,27 @@ describe("resolveWorkerToolAuthority", () => { expect(authority({}, true)).toContain("github_publish"); }); + it("exposes portals only for SSH-backed placements and allowed capability policy", () => { + expect(authority()).not.toContain("portal"); + expect(authority({}, false, true)).toContain("portal"); + expect(authority({ config: { tools: { deny: ["portal"] } } }, false, true)).not.toContain( + "portal", + ); + expect( + authority( + { + sessionKey: "agent:main:worker-sandboxed", + config: { + agents: { defaults: { sandbox: { mode: "all" } } }, + tools: { sandbox: { tools: { deny: ["portal"] } } }, + }, + }, + false, + true, + ), + ).not.toContain("portal"); + }); + it("uses scheduled owner group policy without reapplying fresh sender overlays", () => { const config = { tools: { diff --git a/src/gateway/worker-environments/worker-tool-authority.ts b/src/gateway/worker-environments/worker-tool-authority.ts index 3decbc1d7bb5..a59c5508020c 100644 --- a/src/gateway/worker-environments/worker-tool-authority.ts +++ b/src/gateway/worker-environments/worker-tool-authority.ts @@ -76,6 +76,7 @@ export function resolveWorkerToolAuthority(params: { turn: SessionPlacementTurnParams; availableOptionalToolNames?: readonly WorkerOptionalLocalToolName[]; githubPublicationAvailable?: boolean; + portalAvailable?: boolean; }): WorkerToolAuthority { const turn = params.turn; if (turn.disableTools === true || turn.modelRun === true || turn.promptMode === "none") { @@ -85,8 +86,10 @@ export function resolveWorkerToolAuthority(params: { [ ...WORKER_REQUIRED_LOCAL_TOOL_NAMES, ...(params.availableOptionalToolNames ?? []), - ...WORKER_SESSION_TOOL_NAMES.filter( - (name) => name !== "github_publish" || params.githubPublicationAvailable === true, + ...WORKER_SESSION_TOOL_NAMES.filter((name) => + name === "github_publish" + ? params.githubPublicationAvailable === true + : name !== "portal" || params.portalAvailable === true, ), ].map((name) => ({ name })), turn.toolsAllow, diff --git a/src/gateway/worker-environments/worker-turn-failure.ts b/src/gateway/worker-environments/worker-turn-failure.ts index b24c82c30454..94580a55e20f 100644 --- a/src/gateway/worker-environments/worker-turn-failure.ts +++ b/src/gateway/worker-environments/worker-turn-failure.ts @@ -18,7 +18,7 @@ export type WorkerTurnEnvironmentService = Pick< | "startTunnel" | "stopTunnel" > & - Partial>; + Partial>; export type ActiveWorkerPlacement = Extract; diff --git a/src/gateway/worker-environments/worker-turn-launcher-claim-admission.test.ts b/src/gateway/worker-environments/worker-turn-launcher-claim-admission.test.ts index a0cef62278bf..34cfcf207f58 100644 --- a/src/gateway/worker-environments/worker-turn-launcher-claim-admission.test.ts +++ b/src/gateway/worker-environments/worker-turn-launcher-claim-admission.test.ts @@ -315,7 +315,10 @@ describe("worker turn launcher claim admission", () => { } }); - it("launches only one worker loop for concurrent admission of the same run", async () => { + it.each([ + { label: "without node portal support", portalAvailable: false }, + { label: "with negotiated node portal support", portalAvailable: true }, + ])("launches one worker loop $label", async ({ portalAvailable }) => { seedActivePlacement(); const commandStarted = createDeferred(); const commandFinished = createDeferred<{ @@ -332,7 +335,12 @@ describe("worker turn launcher claim admission", () => { return commandFinished.promise; }); const environments: WorkerTurnEnvironmentService = { - get: vi.fn(() => attachedEnvironment()), + get: vi.fn(() => ({ + ...attachedEnvironment(), + nodeDeviceId: "cloud-node-1", + sshEndpoint: null, + })), + supportsNodePortal: vi.fn(async () => portalAvailable), acquireTurnCredential: vi.fn(async () => credential()), acknowledgeCredentialDelivery: vi.fn(() => true), startTunnel: vi.fn(async () => ({ @@ -396,6 +404,10 @@ describe("worker turn launcher claim admission", () => { permissionMode: "workspace", workerContainmentRoot: "/worker/workspace", }); + expect(launchRequest.plan.assignment.toolAuthority.allowedToolNames.includes("portal")).toBe( + portalAvailable, + ); + expect(environments.supportsNodePortal).toHaveBeenCalledWith(ENVIRONMENT_ID, OWNER_EPOCH); createWorkerSessionPlacementGate(placements).updateAckCursors({ claim: launchRequest.turnClaim, transcriptSeq: 2, diff --git a/src/gateway/worker-environments/worker-turn-launcher.ts b/src/gateway/worker-environments/worker-turn-launcher.ts index ffe066ec69e8..27c913927314 100644 --- a/src/gateway/worker-environments/worker-turn-launcher.ts +++ b/src/gateway/worker-environments/worker-turn-launcher.ts @@ -207,12 +207,20 @@ async function executeWorkerTurn(params: { ...(turn.abortSignal ? { signal: turn.abortSignal } : {}), timeoutMs: turn.timeoutMs, }); + const portalAvailable = + Boolean(environment.nodeDeviceId) && + environment.sshEndpoint === null && + (await params.environments.supportsNodePortal?.( + placement.environmentId, + placement.activeOwnerEpoch, + )) === true; const reasoning = mapThinkingLevelForProvider(turn.thinkLevel); const { browser, toolAuthority } = resolveWorkerBrowserLaunchPlan({ desktop: environment.desktop, modelRef, turn, githubPublicationAvailable, + portalAvailable, }); params.placements.authorizeWorkerTurnTools(params.turnClaim, toolAuthority.allowedToolNames); const { operationalRunInstance, runtimeIdentity } = await prepareWorkerAgentRuntimeIdentity({ diff --git a/src/gateway/worker-environments/worker-turn-rpc.portal.test.ts b/src/gateway/worker-environments/worker-turn-rpc.portal.test.ts new file mode 100644 index 000000000000..c0c294eef942 --- /dev/null +++ b/src/gateway/worker-environments/worker-turn-rpc.portal.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it, vi } from "vitest"; +import * as support from "./service.test-support.js"; + +describe("worker portal RPC authority", () => { + support.setupWorkerEnvironmentServiceSuite(); + + it("executes portal requests only while the exact worker turn remains authorized", async () => { + const result = { resultJson: '{"ok":true}' }; + const executeSessionTool = vi + .fn>() + .mockResolvedValue(result); + const { identity, placementStore, workerService } = support.placementHarness( + "worker-portal-authority", + "session-portal-authority", + { executeSessionTool }, + ); + const request = { toolCallId: "portal-call", action: "open" as const, port: 3000 }; + + await expect(workerService.executeSessionTool(identity, "portal", request)).resolves.toEqual({ + ok: true, + result, + }); + expect(executeSessionTool).toHaveBeenCalledWith({ identity, toolName: "portal", request }); + + placementStore.isWorkerTurnToolAuthorized.mockReturnValue(false); + await expect(workerService.executeSessionTool(identity, "portal", request)).resolves.toEqual({ + ok: false, + closeReason: "method-not-allowed", + }); + expect(executeSessionTool).toHaveBeenCalledOnce(); + + placementStore.isWorkerTurnToolAuthorized.mockReturnValue(true); + executeSessionTool.mockImplementationOnce(async () => { + placementStore.validateWorkerTurn.mockReturnValue(false); + return result; + }); + await expect(workerService.executeSessionTool(identity, "portal", request)).resolves.toEqual({ + ok: false, + closeReason: "placement-mismatch", + }); + }); +}); diff --git a/src/gateway/worker-environments/worker-turn-rpc.ts b/src/gateway/worker-environments/worker-turn-rpc.ts index ded570ff0d2f..24848f7effe0 100644 --- a/src/gateway/worker-environments/worker-turn-rpc.ts +++ b/src/gateway/worker-environments/worker-turn-rpc.ts @@ -3,6 +3,7 @@ import type { WorkerGitHubPublishParams, WorkerConnectParams, WorkerLiveEventParams, + WorkerPortalParams, WorkerProtocolCloseReason, WorkerSessionsSendParams, WorkerSessionsSpawnParams, @@ -118,6 +119,12 @@ type WorkerTurnRpcOptions = { toolName: "github_publish"; request: WorkerGitHubPublishParams; signal?: AbortSignal; + } + | { + identity: WorkerConnectionIdentity; + toolName: "portal"; + request: WorkerPortalParams; + signal?: AbortSignal; }, ) => Promise; inference: ReturnType; @@ -386,7 +393,11 @@ export function createWorkerTurnRpc(options: WorkerTurnRpcOptions) { const executeSessionTool = async ( identity: WorkerConnectionIdentity, toolName: WorkerSessionToolName, - request: WorkerSessionsSpawnParams | WorkerSessionsSendParams | WorkerGitHubPublishParams, + request: + | WorkerSessionsSpawnParams + | WorkerSessionsSendParams + | WorkerGitHubPublishParams + | WorkerPortalParams, signal?: AbortSignal, ): Promise => { const validate = () => { @@ -411,30 +422,26 @@ export function createWorkerTurnRpc(options: WorkerTurnRpcOptions) { if (!options.executeSessionTool) { return { ok: false, reason: "gateway-unavailable" }; } + const operation = + toolName === "sessions_spawn" && "task" in request + ? { toolName, request } + : toolName === "sessions_send" && "sessionKey" in request + ? { toolName, request } + : toolName === "portal" && "action" in request + ? { toolName, request } + : toolName === "github_publish" + ? { toolName, request } + : undefined; + if (!operation) { + return { ok: false, closeReason: "invalid-frame" }; + } let result: WorkerSessionToolResult; try { - result = await options.executeSessionTool( - toolName === "sessions_spawn" - ? { - identity, - toolName, - request: request as WorkerSessionsSpawnParams, - ...(signal ? { signal } : {}), - } - : toolName === "sessions_send" - ? { - identity, - toolName, - request: request as WorkerSessionsSendParams, - ...(signal ? { signal } : {}), - } - : { - identity, - toolName, - request, - ...(signal ? { signal } : {}), - }, - ); + result = await options.executeSessionTool({ + identity, + ...operation, + ...(signal ? { signal } : {}), + }); } catch { return { ok: false, reason: "gateway-unavailable" }; } diff --git a/src/infra/node-commands.ts b/src/infra/node-commands.ts index d389899cc3af..81433f4f0cf5 100644 --- a/src/infra/node-commands.ts +++ b/src/infra/node-commands.ts @@ -27,6 +27,7 @@ export const NODE_WORKER_WORKSPACE_EXEC_COMMAND = "worker.workspace.exec.v1"; export const NODE_WORKER_WORKSPACE_RETAIN_COMMAND = "worker.workspace.retain.v1"; export const NODE_WORKER_DESKTOP_STREAM_COMMAND = "worker.desktop.stream.v1"; export const NODE_WORKER_DESKTOP_LAUNCH_COMMAND = "worker.desktop.launch.v1"; +export const NODE_WORKER_PORTAL_STREAM_COMMAND = "worker.portal.stream.v1"; export const NODE_WORKER_CAPACITY_EXHAUSTED_ERROR_CODE = "WORKER_CAPACITY_EXHAUSTED"; export const NODE_WORKER_PRIVATE_COMMANDS = [ NODE_WORKER_BUNDLE_INSTALL_COMMAND, @@ -37,6 +38,7 @@ export const NODE_WORKER_PRIVATE_COMMANDS = [ NODE_WORKER_WORKSPACE_RETAIN_COMMAND, NODE_WORKER_DESKTOP_STREAM_COMMAND, NODE_WORKER_DESKTOP_LAUNCH_COMMAND, + NODE_WORKER_PORTAL_STREAM_COMMAND, ] as const; const PRIVATE_NODE_INVOKE_COMMAND_SET = new Set(NODE_WORKER_PRIVATE_COMMANDS); diff --git a/src/infra/node-runner-inventory.ts b/src/infra/node-runner-inventory.ts index f903c8119e28..f2970501478e 100644 --- a/src/infra/node-runner-inventory.ts +++ b/src/infra/node-runner-inventory.ts @@ -12,6 +12,7 @@ const RETIRED_NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURES = [ ] as const; export const NODE_WORKER_BUNDLE_RETENTION_VERSION = 1; export const NODE_WORKER_BUNDLE_STATUS_VERSION = 1; +export const NODE_WORKER_PORTAL_STREAM_VERSION = 1; export const NODE_WORKER_CAPACITY_MAX = 1_024; export const NODE_RUNNER_UPDATE_REQUIRED_ISSUE = { @@ -35,6 +36,7 @@ export type NodeWorkerHostDeclaration = bundlePrewarm?: typeof WORKER_BUNDLE_PREWARM_VERSION; bundleRetention?: typeof NODE_WORKER_BUNDLE_RETENTION_VERSION; bundleStatus?: typeof NODE_WORKER_BUNDLE_STATUS_VERSION; + portalStream?: typeof NODE_WORKER_PORTAL_STREAM_VERSION; }; export type NodeRunnerInventoryDeclaration = @@ -83,7 +85,7 @@ function parseWorkerHostDeclaration(value: unknown): NodeWorkerHostDeclaration | if ( !capacity || keys.length < 2 || - keys.length > 5 || + keys.length > 6 || !keys.includes("enabled") || !keys.includes("capacity") || keys.some( @@ -92,13 +94,16 @@ function parseWorkerHostDeclaration(value: unknown): NodeWorkerHostDeclaration | key !== "capacity" && key !== "bundlePrewarm" && key !== "bundleRetention" && - key !== "bundleStatus", + key !== "bundleStatus" && + key !== "portalStream", ) || (value.bundlePrewarm !== undefined && value.bundlePrewarm !== WORKER_BUNDLE_PREWARM_VERSION) || (value.bundleRetention !== undefined && value.bundleRetention !== NODE_WORKER_BUNDLE_RETENTION_VERSION) || (value.bundleStatus !== undefined && value.bundleStatus !== NODE_WORKER_BUNDLE_STATUS_VERSION) || + (value.portalStream !== undefined && + value.portalStream !== NODE_WORKER_PORTAL_STREAM_VERSION) || (value.bundleStatus !== undefined && value.bundleRetention === undefined) ) { return null; @@ -115,6 +120,9 @@ function parseWorkerHostDeclaration(value: unknown): NodeWorkerHostDeclaration | ...(value.bundleStatus === NODE_WORKER_BUNDLE_STATUS_VERSION ? { bundleStatus: NODE_WORKER_BUNDLE_STATUS_VERSION } : {}), + ...(value.portalStream === NODE_WORKER_PORTAL_STREAM_VERSION + ? { portalStream: NODE_WORKER_PORTAL_STREAM_VERSION } + : {}), }; } diff --git a/src/node-host/desktop-stream-command.ts b/src/node-host/desktop-stream-command.ts index 9bb98353a25b..b206ae526e11 100644 --- a/src/node-host/desktop-stream-command.ts +++ b/src/node-host/desktop-stream-command.ts @@ -1,25 +1,16 @@ import { constants as fsConstants } from "node:fs"; import fs from "node:fs/promises"; -import net from "node:net"; -import type { TLSSocket } from "node:tls"; import { isRecord } from "@openclaw/normalization-core/record-coerce"; -import { WebSocket, type ClientOptions, type RawData } from "ws"; -import { normalizeTlsFingerprint } from "../../packages/gateway-client/src/client-address-utils.js"; -import { - buildCloudflareAccessHeaders, - type CloudflareAccessCredentials, -} from "../../packages/gateway-client/src/cloudflare-access.js"; +import type { CloudflareAccessCredentials } from "../../packages/gateway-client/src/cloudflare-access.js"; import type { DesktopHostConfig } from "../config/types.desktop.js"; import { classifyRfbSecurity, probeRfbServer } from "../gateway/desktop/rfb-probe.js"; import { registerSecretValueForRedaction } from "../logging/secret-redaction-registry.js"; import { NODE_DESKTOP_ATTACH_PATH } from "../shared/node-desktop-stream.js"; import { parseNodeWorkerDesktopStreamInput } from "../worker/node-desktop-protocol.js"; +import { runNodeStreamTransport } from "./node-stream-transport.js"; const DEFAULT_DESKTOP_PORT = 5900; const PROBE_TIMEOUT_MS = 1_500; -const MAX_PAYLOAD_BYTES = 1024 * 1024; -const PAUSE_BUFFERED_BYTES = 4 * 1024 * 1024; -const RESUME_CHECK_MS = 25; const TICKET_PATTERN = /^[a-f0-9]{48}$/u; const MAX_VNC_PASSWORD_BYTES = 4 * 1024; @@ -61,89 +52,6 @@ function decodeDesktopStreamParams(raw?: string | null): NodeDesktopStreamComman return { ticket, attachPath }; } -function websocketDataBuffer(data: RawData): Buffer { - if (Buffer.isBuffer(data)) { - return data; - } - if (Array.isArray(data)) { - return Buffer.concat(data); - } - return Buffer.from(data); -} - -function attachWebSocketUrl(gatewayUrl: string, attachPath: string): string { - const gateway = new URL(gatewayUrl); - const url = new URL(attachPath, gateway); - if (url.protocol !== "ws:" && url.protocol !== "wss:") { - throw new Error("desktop stream gateway URL must use WebSocket transport"); - } - if (url.origin !== gateway.origin || url.pathname !== NODE_DESKTOP_ATTACH_PATH) { - throw new Error("desktop stream attachPath must stay on the connected gateway"); - } - return url.toString(); -} - -function assertTlsSocketFingerprint(socket: TLSSocket, expectedRaw: string): void { - const expected = normalizeTlsFingerprint(expectedRaw); - const actual = normalizeTlsFingerprint(socket.getPeerCertificate().fingerprint256 ?? ""); - if (!expected || !actual || actual !== expected) { - throw new Error("gateway TLS fingerprint mismatch"); - } -} - -function createPinnedRequestFinisher( - expected: string, -): NonNullable { - return (request) => { - request.once("socket", (socket) => { - const tlsSocket = socket as TLSSocket; - tlsSocket.once("secureConnect", () => { - try { - assertTlsSocketFingerprint(tlsSocket, expected); - request.end(); - } catch (error) { - request.destroy(error instanceof Error ? error : new Error(String(error))); - } - }); - }); - }; -} - -function websocketOptions( - url: string, - tlsFingerprint?: string, - cloudflareAccess?: CloudflareAccessCredentials, -): ClientOptions { - const edgeHeaders = cloudflareAccess - ? { headers: buildCloudflareAccessHeaders(cloudflareAccess) } - : {}; - if (!url.startsWith("wss:") || !tlsFingerprint?.trim()) { - return { maxPayload: MAX_PAYLOAD_BYTES, ...edgeHeaders }; - } - return { - maxPayload: MAX_PAYLOAD_BYTES, - ...edgeHeaders, - rejectUnauthorized: false, - finishRequest: createPinnedRequestFinisher(tlsFingerprint), - }; -} - -function assertGatewayTlsFingerprint(ws: WebSocket, expectedRaw?: string): void { - if (!expectedRaw?.trim()) { - return; - } - const expected = normalizeTlsFingerprint(expectedRaw); - const socket = ( - ws as WebSocket & { - _socket?: { getPeerCertificate?: () => { fingerprint256?: string } }; - } - )["_socket"]; - const actual = normalizeTlsFingerprint(socket?.getPeerCertificate?.().fingerprint256 ?? ""); - if (!expected || !actual || actual !== expected) { - throw new Error("gateway TLS fingerprint mismatch"); - } -} - async function readVncPassword( passwordFile: string | undefined, signal: AbortSignal, @@ -183,98 +91,6 @@ async function readVncPassword( } } -async function waitForSocketConnect(socket: net.Socket): Promise { - await new Promise((resolve, reject) => { - socket.once("connect", resolve); - socket.once("error", reject); - }); -} - -async function waitForWebSocketOpen(ws: WebSocket): Promise { - await new Promise((resolve, reject) => { - ws.once("open", resolve); - ws.once("error", reject); - }); -} - -async function sendAttachMetadata( - ws: WebSocket, - metadata: { auth: "vnc-password" | "ard-account"; vncPassword?: string }, -): Promise { - const buffer = Buffer.from(JSON.stringify(metadata), "utf8"); - try { - await new Promise((resolve, reject) => { - ws.send(buffer, { binary: true }, (error) => (error ? reject(error) : resolve())); - }); - } finally { - buffer.fill(0); - } -} - -function createDesktopStreamSplice(params: { rfbSocket: net.Socket; ws: WebSocket }) { - let resumeTimer: ReturnType | undefined; - let settled = false; - let finish!: (error?: Error) => void; - const done = new Promise((resolve, reject) => { - finish = (error?: Error) => { - if (settled) { - return; - } - settled = true; - clearInterval(resumeTimer); - if (error) { - reject(error); - } else { - resolve(); - } - }; - params.ws.on("message", (data, isBinary) => { - if (!isBinary) { - finish(new Error("gateway sent non-binary desktop stream data")); - return; - } - if (!params.rfbSocket.write(websocketDataBuffer(data))) { - params.ws.pause(); - params.rfbSocket.once("drain", () => params.ws.resume()); - } - }); - params.rfbSocket.on("data", (chunk) => { - if (params.ws.readyState !== WebSocket.OPEN) { - return; - } - params.ws.send(chunk, { binary: true }, (error) => error && finish(error)); - if (params.ws.bufferedAmount <= PAUSE_BUFFERED_BYTES || resumeTimer) { - return; - } - params.rfbSocket.pause(); - resumeTimer = setInterval(() => { - if (params.ws.bufferedAmount <= PAUSE_BUFFERED_BYTES) { - clearInterval(resumeTimer); - resumeTimer = undefined; - params.rfbSocket.resume(); - } - }, RESUME_CHECK_MS); - resumeTimer.unref?.(); - }); - params.ws.once("close", () => finish()); - params.ws.once("error", (error) => finish(error)); - params.rfbSocket.once("close", () => finish()); - params.rfbSocket.once("error", (error) => finish(error)); - }); - void done.catch(() => undefined); - return { - done, - start() { - if (params.rfbSocket.destroyed || params.ws.readyState !== WebSocket.OPEN) { - finish(); - return; - } - params.rfbSocket.resume(); - params.ws.resume(); - }, - }; -} - /** Splices a node-local loopback RFB socket to a ticket-authenticated Gateway WebSocket. */ async function runNodeDesktopStreamCommand(params: { command: NodeDesktopStreamCommandParams; @@ -322,56 +138,18 @@ async function runNodeDesktopStreamCommand(params: { return; } - const rfbSocket = net.createConnection(params.target.port, "127.0.0.1"); - // The RFB server can send its banner as soon as TCP connects. Pause until the - // attach metadata is accepted so no stateful handshake bytes are lost. - rfbSocket.pause(); - const wsUrl = attachWebSocketUrl(params.gatewayUrl, params.command.attachPath); - const ws = new WebSocket( - wsUrl, - websocketOptions(wsUrl, params.gatewayTlsFingerprint, params.gatewayCloudflareAccess), - ); - let aborted: boolean = params.signal.aborted; - let resolveAbort!: () => void; - const abort = new Promise((resolve) => { - resolveAbort = resolve; + await runNodeStreamTransport({ + gatewayUrl: params.gatewayUrl, + gatewayTlsFingerprint: params.gatewayTlsFingerprint, + gatewayCloudflareAccess: params.gatewayCloudflareAccess, + attachPath: params.command.attachPath, + expectedAttachPath: NODE_DESKTOP_ATTACH_PATH, + port: params.target.port, + metadata: { auth, ...(vncPassword ? { vncPassword } : {}) }, + streamName: "desktop", + signal: params.signal, + emitStatus: params.emitStatus, }); - const onAbort = () => { - aborted = true; - rfbSocket.destroy(); - ws.terminate(); - resolveAbort(); - }; - params.signal.addEventListener("abort", onAbort, { once: true }); - if (aborted) { - onAbort(); - } - try { - await Promise.race([ - Promise.all([waitForSocketConnect(rfbSocket), waitForWebSocketOpen(ws)]), - abort, - ]); - if (aborted) { - return; - } - assertGatewayTlsFingerprint(ws, params.gatewayTlsFingerprint); - ws.pause(); - const splice = createDesktopStreamSplice({ rfbSocket, ws }); - await sendAttachMetadata(ws, { auth, ...(vncPassword ? { vncPassword } : {}) }); - void params.emitStatus?.("desktop stream attached\n").catch(() => undefined); - splice.start(); - await splice.done; - } catch (error) { - if (!aborted) { - throw error; - } - } finally { - params.signal.removeEventListener("abort", onAbort); - rfbSocket.destroy(); - if (ws.readyState === WebSocket.OPEN || ws.readyState === WebSocket.CONNECTING) { - ws.close(); - } - } } /** Runs the built-in command against the node-local desktop configuration. */ diff --git a/src/node-host/invoke-worker-supervisor.test.ts b/src/node-host/invoke-worker-supervisor.test.ts index 4548070649a4..6e635bc77092 100644 --- a/src/node-host/invoke-worker-supervisor.test.ts +++ b/src/node-host/invoke-worker-supervisor.test.ts @@ -8,6 +8,7 @@ import { NODE_WORKER_CAPACITY_EXHAUSTED_ERROR_CODE, NODE_WORKER_DESKTOP_LAUNCH_COMMAND, NODE_WORKER_DESKTOP_STREAM_COMMAND, + NODE_WORKER_PORTAL_STREAM_COMMAND, NODE_WORKER_SUPERVISOR_CANCEL_COMMAND, NODE_WORKER_SUPERVISOR_LAUNCH_COMMAND, NODE_WORKER_SUPERVISOR_STATUS_COMMAND, @@ -207,33 +208,34 @@ describe("node-host worker supervisor commands", () => { expect(payload).not.toHaveProperty("errorText"); }); - it.each([NODE_WORKER_DESKTOP_STREAM_COMMAND, NODE_WORKER_DESKTOP_LAUNCH_COMMAND])( - "dispatches %s before a colliding plugin command", - async (command) => { - const supervisor = supervisorWith(fullReceipt()); - const pluginHandle = vi.fn(async () => '{"plugin":true}'); - const registry = createEmptyPluginRegistry(); - registry.nodeHostCommands = [ - { - pluginId: "malicious", - pluginName: "Malicious", - command: { command, handle: pluginHandle }, - source: "test", - }, - ]; - setActivePluginRegistry(registry); + it.each([ + NODE_WORKER_DESKTOP_STREAM_COMMAND, + NODE_WORKER_DESKTOP_LAUNCH_COMMAND, + NODE_WORKER_PORTAL_STREAM_COMMAND, + ])("dispatches %s before a colliding plugin command", async (command) => { + const supervisor = supervisorWith(fullReceipt()); + const pluginHandle = vi.fn(async () => '{"plugin":true}'); + const registry = createEmptyPluginRegistry(); + registry.nodeHostCommands = [ + { + pluginId: "malicious", + pluginName: "Malicious", + command: { command, handle: pluginHandle }, + source: "test", + }, + ]; + setActivePluginRegistry(registry); - const { result } = await invokePrivate({ - command, - paramsJSON: "{}", - supervisor, - signal: new AbortController().signal, - }); + const { result } = await invokePrivate({ + command, + paramsJSON: "{}", + supervisor, + signal: new AbortController().signal, + }); - expect(pluginHandle).not.toHaveBeenCalled(); - expect(result).toMatchObject({ ok: false, error: { code: "INVALID_REQUEST" } }); - }, - ); + expect(pluginHandle).not.toHaveBeenCalled(); + expect(result).toMatchObject({ ok: false, error: { code: "INVALID_REQUEST" } }); + }); it.each([ { diff --git a/src/node-host/node-stream-transport.ts b/src/node-host/node-stream-transport.ts new file mode 100644 index 000000000000..b1a117bc1e0d --- /dev/null +++ b/src/node-host/node-stream-transport.ts @@ -0,0 +1,272 @@ +import net from "node:net"; +import { TLSSocket } from "node:tls"; +import { WebSocket, type ClientOptions, type RawData } from "ws"; +import { normalizeTlsFingerprint } from "../../packages/gateway-client/src/client-address-utils.js"; +import { + buildCloudflareAccessHeaders, + type CloudflareAccessCredentials, +} from "../../packages/gateway-client/src/cloudflare-access.js"; + +const MAX_PAYLOAD_BYTES = 1024 * 1024; +const PAUSE_BUFFERED_BYTES = 4 * 1024 * 1024; +const RESUME_CHECK_MS = 25; + +function websocketDataBuffer(data: RawData): Buffer { + if (Buffer.isBuffer(data)) { + return data; + } + if (Array.isArray(data)) { + return Buffer.concat(data); + } + return Buffer.from(data); +} + +function attachWebSocketUrl(params: { + gatewayUrl: string; + attachPath: string; + expectedAttachPath: string; + streamName: string; +}): string { + const gateway = new URL(params.gatewayUrl); + const url = new URL(params.attachPath, gateway); + if (url.protocol !== "ws:" && url.protocol !== "wss:") { + throw new Error(`${params.streamName} stream gateway URL must use WebSocket transport`); + } + if (url.origin !== gateway.origin || url.pathname !== params.expectedAttachPath) { + throw new Error(`${params.streamName} stream attachPath must stay on the connected gateway`); + } + return url.toString(); +} + +function assertTlsSocketFingerprint(socket: TLSSocket, expectedRaw: string): void { + const expected = normalizeTlsFingerprint(expectedRaw); + const actual = normalizeTlsFingerprint(socket.getPeerCertificate().fingerprint256 ?? ""); + if (!expected || !actual || actual !== expected) { + throw new Error("gateway TLS fingerprint mismatch"); + } +} + +function createPinnedRequestFinisher( + expected: string, +): NonNullable { + return (request) => { + request.once("socket", (socket) => { + if (!(socket instanceof TLSSocket)) { + request.destroy(new Error("gateway TLS fingerprint mismatch")); + return; + } + socket.once("secureConnect", () => { + try { + assertTlsSocketFingerprint(socket, expected); + request.end(); + } catch (error) { + request.destroy(error instanceof Error ? error : new Error(String(error))); + } + }); + }); + }; +} + +function websocketOptions( + url: string, + tlsFingerprint?: string, + cloudflareAccess?: CloudflareAccessCredentials, +): ClientOptions { + const edgeHeaders = cloudflareAccess + ? { headers: buildCloudflareAccessHeaders(cloudflareAccess) } + : {}; + if (!url.startsWith("wss:") || !tlsFingerprint?.trim()) { + return { maxPayload: MAX_PAYLOAD_BYTES, ...edgeHeaders }; + } + return { + maxPayload: MAX_PAYLOAD_BYTES, + ...edgeHeaders, + rejectUnauthorized: false, + finishRequest: createPinnedRequestFinisher(tlsFingerprint), + }; +} + +function assertGatewayTlsFingerprint(socket: TLSSocket | undefined, expectedRaw?: string): void { + if (!expectedRaw?.trim()) { + return; + } + const expected = normalizeTlsFingerprint(expectedRaw); + const actual = normalizeTlsFingerprint(socket?.getPeerCertificate().fingerprint256 ?? ""); + if (!expected || !actual || actual !== expected) { + throw new Error("gateway TLS fingerprint mismatch"); + } +} + +async function waitForSocketConnect(socket: net.Socket): Promise { + await new Promise((resolve, reject) => { + socket.once("connect", resolve); + socket.once("error", reject); + }); +} + +async function waitForWebSocketOpen(ws: WebSocket): Promise { + await new Promise((resolve, reject) => { + ws.once("open", resolve); + ws.once("error", reject); + }); +} + +async function sendAttachMetadata( + ws: WebSocket, + metadata: Record, +): Promise { + const buffer = Buffer.from(JSON.stringify(metadata), "utf8"); + try { + await new Promise((resolve, reject) => { + ws.send(buffer, { binary: true }, (error) => (error ? reject(error) : resolve())); + }); + } finally { + buffer.fill(0); + } +} + +function createNodeStreamSplice(params: { socket: net.Socket; ws: WebSocket; streamName: string }) { + let resumeTimer: ReturnType | undefined; + let settled = false; + let finish!: (error?: Error) => void; + const done = new Promise((resolve, reject) => { + finish = (error?: Error) => { + if (settled) { + return; + } + settled = true; + clearInterval(resumeTimer); + if (error) { + reject(error); + } else { + resolve(); + } + }; + params.ws.on("message", (data, isBinary) => { + if (!isBinary) { + finish(new Error(`gateway sent non-binary ${params.streamName} stream data`)); + return; + } + if (!params.socket.write(websocketDataBuffer(data))) { + params.ws.pause(); + params.socket.once("drain", () => params.ws.resume()); + } + }); + params.socket.on("data", (chunk) => { + if (params.ws.readyState !== WebSocket.OPEN) { + return; + } + params.ws.send(chunk, { binary: true }, (error) => error && finish(error)); + if (params.ws.bufferedAmount <= PAUSE_BUFFERED_BYTES || resumeTimer) { + return; + } + params.socket.pause(); + resumeTimer = setInterval(() => { + if (params.ws.bufferedAmount <= PAUSE_BUFFERED_BYTES) { + clearInterval(resumeTimer); + resumeTimer = undefined; + params.socket.resume(); + } + }, RESUME_CHECK_MS); + resumeTimer.unref?.(); + }); + params.ws.once("close", () => finish()); + params.ws.once("error", (error) => finish(error)); + params.socket.once("close", () => finish()); + params.socket.once("error", (error) => finish(error)); + }); + void done.catch(() => undefined); + return { + done, + start() { + if (params.socket.destroyed || params.ws.readyState !== WebSocket.OPEN) { + finish(); + return; + } + params.socket.resume(); + params.ws.resume(); + }, + }; +} + +/** Pairs an enrolled Gateway attach socket with a node-owned loopback connection. */ +export async function runNodeStreamTransport(params: { + gatewayUrl: string; + gatewayTlsFingerprint?: string; + gatewayCloudflareAccess?: CloudflareAccessCredentials; + attachPath: string; + expectedAttachPath: string; + port: number; + metadata: Record; + streamName: string; + signal: AbortSignal; + connectAfterGatewayAttach?: boolean; + emitStatus?: (status: string) => Promise; +}): Promise { + const socket = params.connectAfterGatewayAttach + ? new net.Socket() + : net.createConnection(params.port, "127.0.0.1"); + // Loopback peers may send immediately; retain their first bytes until metadata is accepted. + socket.pause(); + const wsUrl = attachWebSocketUrl(params); + const ws = new WebSocket( + wsUrl, + websocketOptions(wsUrl, params.gatewayTlsFingerprint, params.gatewayCloudflareAccess), + ); + let gatewayTlsSocket: TLSSocket | undefined; + ws.once("upgrade", (response) => { + if (response.socket instanceof TLSSocket) { + gatewayTlsSocket = response.socket; + } + }); + let aborted: boolean = params.signal.aborted; + let resolveAbort!: () => void; + const abort = new Promise((resolve) => { + resolveAbort = resolve; + }); + const onAbort = () => { + aborted = true; + socket.destroy(); + ws.terminate(); + resolveAbort(); + }; + params.signal.addEventListener("abort", onAbort, { once: true }); + if (aborted) { + onAbort(); + } + try { + if (params.connectAfterGatewayAttach) { + // Attach first so a refused target closes a claimed ticket instead of leaving it pending. + await Promise.race([waitForWebSocketOpen(ws), abort]); + if (!aborted) { + socket.connect(params.port, "127.0.0.1"); + await Promise.race([waitForSocketConnect(socket), abort]); + } + } else { + await Promise.race([ + Promise.all([waitForSocketConnect(socket), waitForWebSocketOpen(ws)]), + abort, + ]); + } + if (aborted) { + return; + } + assertGatewayTlsFingerprint(gatewayTlsSocket, params.gatewayTlsFingerprint); + ws.pause(); + const splice = createNodeStreamSplice({ socket, ws, streamName: params.streamName }); + await sendAttachMetadata(ws, params.metadata); + void params.emitStatus?.(`${params.streamName} stream attached\n`).catch(() => undefined); + splice.start(); + await splice.done; + } catch (error) { + if (!aborted) { + throw error; + } + } finally { + params.signal.removeEventListener("abort", onAbort); + socket.destroy(); + if (ws.readyState === WebSocket.OPEN || ws.readyState === WebSocket.CONNECTING) { + ws.close(); + } + } +} diff --git a/src/node-host/node-worker-supervisor-commands.ts b/src/node-host/node-worker-supervisor-commands.ts index 6671e59181f5..13d71f366d76 100644 --- a/src/node-host/node-worker-supervisor-commands.ts +++ b/src/node-host/node-worker-supervisor-commands.ts @@ -5,6 +5,7 @@ import { NODE_WORKER_CAPACITY_EXHAUSTED_ERROR_CODE, NODE_WORKER_DESKTOP_LAUNCH_COMMAND, NODE_WORKER_DESKTOP_STREAM_COMMAND, + NODE_WORKER_PORTAL_STREAM_COMMAND, NODE_WORKER_SUPERVISOR_CANCEL_COMMAND, NODE_WORKER_SUPERVISOR_LAUNCH_COMMAND, NODE_WORKER_SUPERVISOR_STATUS_COMMAND, @@ -46,6 +47,7 @@ import { type NodeWorkerSupervisorReceipt, } from "./node-worker-supervisor-contract.js"; import type { NodeWorkerWorkspaceRuntime } from "./node-worker-workspace.js"; +import { invokeNodeWorkerPortalStream } from "./portal-stream-command.js"; type NodeWorkerSupervisorCommandResult = | { handled: false } @@ -126,7 +128,8 @@ export async function invokeNodeWorkerSupervisorCommand(params: { params.command === NODE_WORKER_WORKSPACE_EXEC_COMMAND || params.command === NODE_WORKER_WORKSPACE_RETAIN_COMMAND || params.command === NODE_WORKER_DESKTOP_STREAM_COMMAND || - params.command === NODE_WORKER_DESKTOP_LAUNCH_COMMAND; + params.command === NODE_WORKER_DESKTOP_LAUNCH_COMMAND || + params.command === NODE_WORKER_PORTAL_STREAM_COMMAND; if (!recognized) { return { handled: false }; } @@ -245,6 +248,16 @@ export async function invokeNodeWorkerSupervisorCommand(params: { }); return { handled: true, ok: true, payload: null }; } + if (params.command === NODE_WORKER_PORTAL_STREAM_COMMAND) { + await invokeNodeWorkerPortalStream({ + paramsJSON: params.paramsJSON, + gatewayUrl: params.gatewayUrl, + gatewayTlsFingerprint: params.gatewayTlsFingerprint, + gatewayCloudflareAccess: params.gatewayCloudflareAccess, + signal: params.signal, + }); + return { handled: true, ok: true, payload: null }; + } if (params.command === NODE_WORKER_DESKTOP_LAUNCH_COMMAND) { return { handled: true, diff --git a/src/node-host/portal-stream-command.test.ts b/src/node-host/portal-stream-command.test.ts new file mode 100644 index 000000000000..61a209055ca8 --- /dev/null +++ b/src/node-host/portal-stream-command.test.ts @@ -0,0 +1,181 @@ +import http from "node:http"; +import net from "node:net"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { WebSocket, WebSocketServer } from "ws"; +import { invokeNodeWorkerPortalStream } from "./portal-stream-command.js"; + +const TICKET = "a".repeat(48); +const cleanups: Array<() => Promise> = []; + +async function listenGateway(onConnection: (ws: WebSocket, request: http.IncomingMessage) => void) { + const server = http.createServer(); + const wss = new WebSocketServer({ server }); + wss.on("connection", onConnection); + await new Promise((resolve) => { + server.listen(0, "127.0.0.1", resolve); + }); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("expected Gateway test address"); + } + cleanups.push( + async () => + await new Promise((resolve) => { + for (const client of wss.clients) { + client.terminate(); + } + wss.close(() => server.close(() => resolve())); + }), + ); + return `ws://127.0.0.1:${address.port}`; +} + +function portalCommand(port: number) { + return JSON.stringify({ + ticket: TICKET, + attachPath: `/node-portal/attach?ticket=${TICKET}`, + port, + }); +} + +afterEach(async () => { + await Promise.all(cleanups.splice(0).map((cleanup) => cleanup())); +}); + +describe("node worker portal stream command", () => { + it.each([ + ["missing ticket", { ticket: undefined }], + ["invalid ticket", { ticket: "invalid" }], + ["cross-origin attach path", { attachPath: `//attacker.example/attach?ticket=${TICKET}` }], + ["desktop attach path", { attachPath: `/node-desktop/attach?ticket=${TICKET}` }], + ["invalid port", { port: 65_536 }], + ["caller-selected host", { host: "192.0.2.10" }], + ])("rejects a request with %s", async (_name, override) => { + await expect( + invokeNodeWorkerPortalStream({ + paramsJSON: JSON.stringify({ + ticket: TICKET, + attachPath: `/node-portal/attach?ticket=${TICKET}`, + port: 8080, + ...override, + }), + gatewayUrl: "ws://127.0.0.1:1", + signal: new AbortController().signal, + }), + ).rejects.toThrow("INVALID_REQUEST"); + }); + + it("attaches an enrolled loopback socket, exchanges binary data, and closes on cancellation", async () => { + const peers = new Set(); + const local = net.createServer((socket) => { + peers.add(socket); + socket.once("close", () => peers.delete(socket)); + socket.on("data", (chunk) => + socket.write(Buffer.concat([Buffer.from("echo:"), Buffer.from(chunk)])), + ); + }); + await new Promise((resolve) => { + local.listen(0, "127.0.0.1", resolve); + }); + const address = local.address(); + if (!address || typeof address === "string") { + throw new Error("expected portal loopback test address"); + } + cleanups.push( + async () => + await new Promise((resolve) => { + for (const peer of peers) { + peer.destroy(); + } + local.close(() => resolve()); + }), + ); + + const frames: Buffer[] = []; + let attached: WebSocket | undefined; + let accessHeaders: [string | undefined, string | undefined] | undefined; + let closed = false; + const gatewayUrl = await listenGateway((ws, request) => { + attached = ws; + accessHeaders = [ + request.headers["cf-access-client-id"]?.toString(), + request.headers["cf-access-client-secret"]?.toString(), + ]; + ws.on("message", (data, isBinary) => { + expect(isBinary).toBe(true); + if (!Buffer.isBuffer(data)) { + throw new Error("expected binary portal stream frame"); + } + frames.push(data); + }); + ws.once("close", () => { + closed = true; + }); + }); + const controller = new AbortController(); + const running = invokeNodeWorkerPortalStream({ + paramsJSON: portalCommand(address.port), + gatewayUrl, + gatewayCloudflareAccess: { + clientId: "portal-client-id", + clientSecret: "portal-client-secret", + }, + signal: controller.signal, + }); + + await vi.waitFor(() => expect(frames).toHaveLength(1)); + expect(frames[0]?.toString("utf8")).toBe(JSON.stringify({ ok: true })); + expect(accessHeaders).toEqual(["portal-client-id", "portal-client-secret"]); + attached?.send(Buffer.from("hello"), { binary: true }); + await vi.waitFor(() => expect(frames[1]?.toString("utf8")).toBe("echo:hello")); + + controller.abort(); + + await expect(running).resolves.toBeUndefined(); + await vi.waitFor(() => expect(closed).toBe(true)); + await vi.waitFor(() => expect(peers.size).toBe(0)); + }); + + it("closes an attached Gateway socket without a readiness frame when loopback refuses", async () => { + const unavailable = net.createServer(); + await new Promise((resolve) => { + unavailable.listen(0, "127.0.0.1", resolve); + }); + const address = unavailable.address(); + if (!address || typeof address === "string") { + throw new Error("expected unavailable portal test address"); + } + await new Promise((resolve, reject) => { + unavailable.close((error) => { + if (error) { + reject(error); + return; + } + resolve(); + }); + }); + + let attached = false; + let closed = false; + const frames: unknown[] = []; + const gatewayUrl = await listenGateway((ws) => { + attached = true; + ws.on("message", (data) => frames.push(data)); + ws.once("close", () => { + closed = true; + }); + }); + + await expect( + invokeNodeWorkerPortalStream({ + paramsJSON: portalCommand(address.port), + gatewayUrl, + signal: new AbortController().signal, + }), + ).rejects.toThrow("ECONNREFUSED"); + + expect(attached).toBe(true); + await vi.waitFor(() => expect(closed).toBe(true)); + expect(frames).toEqual([]); + }); +}); diff --git a/src/node-host/portal-stream-command.ts b/src/node-host/portal-stream-command.ts new file mode 100644 index 000000000000..3d91e78de15d --- /dev/null +++ b/src/node-host/portal-stream-command.ts @@ -0,0 +1,63 @@ +import { isRecord } from "@openclaw/normalization-core/record-coerce"; +import type { CloudflareAccessCredentials } from "../../packages/gateway-client/src/cloudflare-access.js"; +import { NODE_PORTAL_ATTACH_PATH } from "../shared/node-desktop-stream.js"; +import { runNodeStreamTransport } from "./node-stream-transport.js"; + +const REQUEST_MAX_BYTES = 16 * 1024; +const TICKET_PATTERN = /^[a-f0-9]{48}$/u; + +function parseNodeWorkerPortalStreamInput(raw?: string | null): { + ticket: string; + attachPath: string; + port: number; +} { + if (!raw || Buffer.byteLength(raw, "utf8") > REQUEST_MAX_BYTES) { + throw new Error("INVALID_REQUEST: invalid node worker portal stream request"); + } + let value: unknown; + try { + value = JSON.parse(raw); + } catch { + throw new Error("INVALID_REQUEST: malformed node worker portal stream request"); + } + if ( + !isRecord(value) || + Object.keys(value).length !== 3 || + typeof value.ticket !== "string" || + !TICKET_PATTERN.test(value.ticket) || + value.attachPath !== `${NODE_PORTAL_ATTACH_PATH}?ticket=${value.ticket}` || + typeof value.port !== "number" || + !Number.isSafeInteger(value.port) || + value.port < 1 || + value.port > 65_535 + ) { + throw new Error("INVALID_REQUEST: invalid node worker portal stream request"); + } + return { ticket: value.ticket, attachPath: value.attachPath, port: value.port }; +} + +/** Runs a private worker portal stream against its exact enrolled Gateway owner. */ +export async function invokeNodeWorkerPortalStream(params: { + paramsJSON?: string | null; + gatewayUrl?: string; + gatewayTlsFingerprint?: string; + gatewayCloudflareAccess?: CloudflareAccessCredentials; + signal?: AbortSignal; +}): Promise { + if (!params.gatewayUrl || !params.signal) { + throw new Error("node worker portal gateway connection is unavailable"); + } + const command = parseNodeWorkerPortalStreamInput(params.paramsJSON); + await runNodeStreamTransport({ + gatewayUrl: params.gatewayUrl, + gatewayTlsFingerprint: params.gatewayTlsFingerprint, + gatewayCloudflareAccess: params.gatewayCloudflareAccess, + attachPath: command.attachPath, + expectedAttachPath: NODE_PORTAL_ATTACH_PATH, + port: command.port, + metadata: { ok: true }, + streamName: "portal", + signal: params.signal, + connectAfterGatewayAttach: true, + }); +} diff --git a/src/node-host/runner.test.ts b/src/node-host/runner.test.ts index f7e6ab194538..44a7db86456b 100644 --- a/src/node-host/runner.test.ts +++ b/src/node-host/runner.test.ts @@ -754,6 +754,14 @@ describe("runNodeHost", () => { }); }); + const negotiatedWorkerHost = { + enabled: true, + capacity: { total: 2, available: 2 }, + bundlePrewarm: 1, + bundleRetention: 1, + bundleStatus: 1, + portalStream: 1, + }; options?.onHelloOk?.({ protocol: 4, features: { @@ -762,19 +770,14 @@ describe("runNodeHost", () => { capabilities: [ GATEWAY_SERVER_CAPS.NODE_WORKER_BUNDLE_RETENTION, GATEWAY_SERVER_CAPS.NODE_WORKER_BUNDLE_STATUS, + GATEWAY_SERVER_CAPS.NODE_WORKER_PORTAL_STREAM, ], }, } as unknown as Parameters>[0]); await vi.waitFor(() => { expect(client?.request).toHaveBeenCalledWith(NODE_RUNNER_INVENTORY_UPDATE_METHOD, { protocolFeatures: [NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE], - workerHost: { - enabled: true, - capacity: { total: 2, available: 2 }, - bundlePrewarm: 1, - bundleRetention: 1, - bundleStatus: 1, - }, + workerHost: negotiatedWorkerHost, }); }); @@ -784,11 +787,8 @@ describe("runNodeHost", () => { expect(client?.request).toHaveBeenLastCalledWith(NODE_RUNNER_INVENTORY_UPDATE_METHOD, { protocolFeatures: [NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE], workerHost: { - enabled: true, + ...negotiatedWorkerHost, capacity: { total: 2, available }, - bundlePrewarm: 1, - bundleRetention: 1, - bundleStatus: 1, }, }); }); diff --git a/src/node-host/runner.ts b/src/node-host/runner.ts index 5dae673ab3a1..ebcabeb639f2 100644 --- a/src/node-host/runner.ts +++ b/src/node-host/runner.ts @@ -20,6 +20,7 @@ import { NODE_RUNNER_INVENTORY_UPDATE_METHOD, NODE_WORKER_BUNDLE_RETENTION_VERSION, NODE_WORKER_BUNDLE_STATUS_VERSION, + NODE_WORKER_PORTAL_STREAM_VERSION, NODE_WORKER_SUPERVISOR_PROTOCOL_FEATURE, type NodeWorkerCapacitySnapshot, } from "../infra/node-runner-inventory.js"; @@ -292,6 +293,7 @@ export async function runNodeHost(opts: NodeHostRunOptions): Promise { let connectedGatewayProtocol = 0; let gatewaySupportsBundleRetention = false; let gatewaySupportsBundleStatus = false; + let gatewaySupportsPortalStream = false; let optionalPublicationStates = new Map< NodeOptionalPublicationMethod, NodeOptionalPublicationState @@ -310,6 +312,7 @@ export async function runNodeHost(opts: NodeHostRunOptions): Promise { connectedGatewayProtocol = 0; gatewaySupportsBundleRetention = false; gatewaySupportsBundleStatus = false; + gatewaySupportsPortalStream = false; retireOptionalPublications(); }; @@ -514,6 +517,9 @@ export async function runNodeHost(opts: NodeHostRunOptions): Promise { ...(gatewaySupportsBundleRetention && gatewaySupportsBundleStatus ? { bundleStatus: NODE_WORKER_BUNDLE_STATUS_VERSION } : {}), + ...(gatewaySupportsPortalStream + ? { portalStream: NODE_WORKER_PORTAL_STREAM_VERSION } + : {}), } : { enabled: false }, }, @@ -598,6 +604,9 @@ export async function runNodeHost(opts: NodeHostRunOptions): Promise { gatewaySupportsBundleStatus = hello.features?.capabilities?.includes(GATEWAY_SERVER_CAPS.NODE_WORKER_BUNDLE_STATUS) === true; + gatewaySupportsPortalStream = + hello.features?.capabilities?.includes(GATEWAY_SERVER_CAPS.NODE_WORKER_PORTAL_STREAM) === + true; retireOptionalPublications(); optionalPublicationStates = new Map(); if (opts.stopAfterFirstConnect) { diff --git a/src/shared/node-desktop-stream.ts b/src/shared/node-desktop-stream.ts index 328d66867cde..859b9eed8a3e 100644 --- a/src/shared/node-desktop-stream.ts +++ b/src/shared/node-desktop-stream.ts @@ -1,2 +1,3 @@ export const NODE_DESKTOP_STREAM_COMMAND = "desktop.stream"; export const NODE_DESKTOP_ATTACH_PATH = "/node-desktop/attach"; +export const NODE_PORTAL_ATTACH_PATH = "/node-portal/attach"; diff --git a/src/worker/tool-authority.ts b/src/worker/tool-authority.ts index 7aad84a6abde..ccdf7a87d50c 100644 --- a/src/worker/tool-authority.ts +++ b/src/worker/tool-authority.ts @@ -19,6 +19,7 @@ export const WORKER_SESSION_TOOL_NAMES = [ "sessions_spawn", "sessions_send", "github_publish", + "portal", ] as const; export const WORKER_TOOL_NAMES = [ diff --git a/src/worker/worker-connection-frames.ts b/src/worker/worker-connection-frames.ts index 6ded4ceecefd..75f1cc773d3e 100644 --- a/src/worker/worker-connection-frames.ts +++ b/src/worker/worker-connection-frames.ts @@ -12,6 +12,9 @@ import { type WorkerLiveEventParams, type WorkerLiveEventResponseFrame, WorkerLiveEventResponseFrameSchema, + type WorkerPortalParams, + type WorkerPortalResponseFrame, + WorkerPortalResponseFrameSchema, WORKER_PROTOCOL_MAX_PAYLOAD_BYTES, type WorkerSessionsSendParams, type WorkerSessionsSendResponseFrame, @@ -71,6 +74,10 @@ const WORKER_REQUEST_SPECS = { method: "worker.github.publish", responseSchema: WorkerGitHubPublishResponseFrameSchema, }, + portal: { + method: "worker.portal", + responseSchema: WorkerPortalResponseFrameSchema, + }, "inference-start": { method: "worker.inference.start", responseSchema: WorkerInferenceStartResponseFrameSchema, @@ -89,6 +96,7 @@ type WorkerRequestParams = { "sessions-spawn": WorkerSessionsSpawnParams; "sessions-send": WorkerSessionsSendParams; "github-publish": WorkerGitHubPublishParams; + portal: WorkerPortalParams; "inference-start": WorkerInferenceStartParams; "inference-cancel": WorkerInferenceCancelParams; }; @@ -99,6 +107,7 @@ type WorkerResponseFrames = { "sessions-spawn": WorkerSessionsSpawnResponseFrame; "sessions-send": WorkerSessionsSendResponseFrame; "github-publish": WorkerGitHubPublishResponseFrame; + portal: WorkerPortalResponseFrame; "inference-start": WorkerInferenceStartResponseFrame; "inference-cancel": WorkerInferenceCancelResponseFrame; }; diff --git a/src/worker/worker-connection.ts b/src/worker/worker-connection.ts index 85f390b09530..6755a02a0612 100644 --- a/src/worker/worker-connection.ts +++ b/src/worker/worker-connection.ts @@ -8,6 +8,8 @@ import type { WorkerHelloOk, WorkerLiveEventParams, WorkerLiveEventResponseFrame, + WorkerPortalParams, + WorkerPortalResponseFrame, WorkerProtocolCloseReason, WorkerSessionsSendParams, WorkerSessionsSendResponseFrame, @@ -233,6 +235,10 @@ export class WorkerConnection { return this.requestDurableSessionOperation(() => this.frames.request("github-publish", params)); } + requestPortal(params: WorkerPortalParams): Promise { + return this.frames.request("portal", params); + } + private async requestDurableSessionOperation(request: () => Promise): Promise { for (;;) { try { diff --git a/src/worker/worker-session-tools.test.ts b/src/worker/worker-session-tools.test.ts index 47d9a4079956..e90535729c6c 100644 --- a/src/worker/worker-session-tools.test.ts +++ b/src/worker/worker-session-tools.test.ts @@ -1,5 +1,10 @@ import { Value } from "typebox/value"; import { describe, expect, it, vi } from "vitest"; +import { + PORTAL_TOOL_DESCRIPTION, + PortalOutputSchema, + PortalToolSchema, +} from "../agents/tools/portal-tool-contract.js"; import { createWorkerSessionTools } from "./worker-session-tools.js"; describe("worker Gateway tools", () => { @@ -17,6 +22,7 @@ describe("worker Gateway tools", () => { })); const tools = createWorkerSessionTools({ requestGitHubPublish, + requestPortal: vi.fn(), requestSessionsSend: vi.fn(), requestSessionsSpawn: vi.fn(), }); @@ -43,4 +49,43 @@ describe("worker Gateway tools", () => { title: "Publish the result", }); }); + + it("forwards portal actions through the shared Gateway portal tool contract", async () => { + const requestPortal = vi.fn(async () => ({ + type: "res" as const, + id: "response-portal", + ok: true as const, + payload: { + resultJson: JSON.stringify({ + content: [{ type: "text", text: "Portal available" }], + details: { id: "worker-portal" }, + }), + }, + })); + const tools = createWorkerSessionTools({ + requestGitHubPublish: vi.fn(), + requestPortal, + requestSessionsSend: vi.fn(), + requestSessionsSpawn: vi.fn(), + }); + const portal = tools.find((candidate) => candidate.name === "portal"); + + expect(portal?.description).toBe(PORTAL_TOOL_DESCRIPTION); + expect(portal?.parameters).toBe(PortalToolSchema); + expect(portal?.outputSchema).toBe(PortalOutputSchema); + expect(Value.Check(PortalToolSchema, { action: "open", port: 3000, path: "/app" })).toBe(true); + expect(Value.Check(PortalToolSchema, { action: "open", port: 0 })).toBe(false); + + await expect(portal?.execute?.("portal-call", { action: "open", port: 3000 })).resolves.toEqual( + { + content: [{ type: "text", text: "Portal available" }], + details: { id: "worker-portal" }, + }, + ); + expect(requestPortal).toHaveBeenCalledWith({ + toolCallId: "portal-call", + action: "open", + port: 3000, + }); + }); }); diff --git a/src/worker/worker-session-tools.ts b/src/worker/worker-session-tools.ts index d5bb76b82eff..f975d8e119da 100644 --- a/src/worker/worker-session-tools.ts +++ b/src/worker/worker-session-tools.ts @@ -1,4 +1,5 @@ import { Type } from "typebox"; +import { Value } from "typebox/value"; import { GitHubPublicationBodySchema, GitHubPublicationTitleSchema, @@ -7,6 +8,9 @@ import { WORKER_SESSION_TOOL_MAX_TEXT_LENGTH, type WorkerGitHubPublishParams, type WorkerGitHubPublishResponseFrame, + type WorkerPortalParams, + type WorkerPortalResponseFrame, + WorkerPortalParamsSchema, type WorkerSessionsSendParams, type WorkerSessionsSendResponseFrame, type WorkerSessionsSpawnParams, @@ -14,6 +18,11 @@ import { } from "../../packages/gateway-protocol/src/schema/worker-admission.js"; import type { AgentToolResult } from "../agents/runtime/index.js"; import type { AnyAgentTool } from "../agents/tools/common.js"; +import { + PORTAL_TOOL_DESCRIPTION, + PortalOutputSchema, + PortalToolSchema, +} from "../agents/tools/portal-tool-contract.js"; type WorkerSessionRpcClient = { requestSessionsSpawn( @@ -23,14 +32,10 @@ type WorkerSessionRpcClient = { requestGitHubPublish( params: WorkerGitHubPublishParams, ): Promise; + requestPortal(params: WorkerPortalParams): Promise; }; -function parseToolResult( - frame: - | WorkerSessionsSpawnResponseFrame - | WorkerSessionsSendResponseFrame - | WorkerGitHubPublishResponseFrame, -) { +function parseToolResult(frame: WorkerSessionsSpawnResponseFrame) { if (!frame.ok) { throw new Error(frame.error.message); } @@ -102,5 +107,22 @@ export function createWorkerSessionTools(client: WorkerSessionRpcClient): AnyAge return parseToolResult(await client.requestSessionsSend({ toolCallId, ...params })); }, }, + { + label: "Portal", + name: "portal", + description: PORTAL_TOOL_DESCRIPTION, + parameters: PortalToolSchema, + outputSchema: PortalOutputSchema, + execute: async (toolCallId, raw) => { + if (!Value.Check(PortalToolSchema, raw)) { + throw new Error("Invalid portal tool arguments"); + } + const params = { toolCallId, ...raw }; + if (!Value.Check(WorkerPortalParamsSchema, params)) { + throw new Error("Portal tool arguments exceed the worker protocol limits"); + } + return parseToolResult(await client.requestPortal(params)); + }, + }, ]; } diff --git a/src/worker/worker.runtime.test.ts b/src/worker/worker.runtime.test.ts index bf7dd5ab6e08..cc78656046b5 100644 --- a/src/worker/worker.runtime.test.ts +++ b/src/worker/worker.runtime.test.ts @@ -10,6 +10,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { WebSocket, WebSocketServer, type RawData } from "ws"; import { validateWorkerGitHubPublishParams, + validateWorkerPortalParams, validateWorkerSessionsSendParams, validateWorkerSessionsSpawnParams, } from "../../packages/gateway-protocol/src/index.js"; @@ -21,9 +22,11 @@ import { type WorkerLiveEventParams, type WorkerLiveEventRequestFrame, WorkerLiveEventRequestFrameSchema, + WORKER_PORTAL_PROTOCOL_FEATURE, WORKER_PROTOCOL_FEATURES, WORKER_RPC_SET_VERSION, type WorkerGitHubPublishParams, + type WorkerPortalParams, type WorkerSessionsSendParams, type WorkerSessionsSpawnParams, type WorkerTranscriptCommitParams, @@ -186,6 +189,7 @@ class FakeWorkerGateway { readonly sessionSpawnRequests: WorkerSessionsSpawnParams[] = []; readonly sessionSendRequests: WorkerSessionsSendParams[] = []; readonly githubPublishRequests: WorkerGitHubPublishParams[] = []; + readonly portalRequests: WorkerPortalParams[] = []; readonly applicationOrder: string[] = []; waitForInferenceStart(): Promise { @@ -280,7 +284,9 @@ class FakeWorkerGateway { : parsed.method === "worker.github.publish" && validateWorkerGitHubPublishParams(parsed.params) ? parsed.method - : undefined; + : parsed.method === "worker.portal" && validateWorkerPortalParams(parsed.params) + ? parsed.method + : undefined; if (sessionToolMethod) { this.handleSessionTool(socket, { id: parsed.id, @@ -288,7 +294,8 @@ class FakeWorkerGateway { params: parsed.params as | WorkerSessionsSpawnParams | WorkerSessionsSendParams - | WorkerGitHubPublishParams, + | WorkerGitHubPublishParams + | WorkerPortalParams, }); return; } @@ -389,8 +396,16 @@ class FakeWorkerGateway { socket: WebSocket, frame: { id: string; - method: "worker.sessions.spawn" | "worker.sessions.send" | "worker.github.publish"; - params: WorkerSessionsSpawnParams | WorkerSessionsSendParams | WorkerGitHubPublishParams; + method: + | "worker.sessions.spawn" + | "worker.sessions.send" + | "worker.github.publish" + | "worker.portal"; + params: + | WorkerSessionsSpawnParams + | WorkerSessionsSendParams + | WorkerGitHubPublishParams + | WorkerPortalParams; }, ): void { this.methods.push(frame.method); @@ -398,13 +413,16 @@ class FakeWorkerGateway { this.sessionSpawnRequests.push(structuredClone(frame.params as WorkerSessionsSpawnParams)); } else if (frame.method === "worker.sessions.send") { this.sessionSendRequests.push(structuredClone(frame.params as WorkerSessionsSendParams)); - } else { + } else if (frame.method === "worker.github.publish") { this.githubPublishRequests.push(structuredClone(frame.params as WorkerGitHubPublishParams)); + } else { + this.portalRequests.push(structuredClone(frame.params as WorkerPortalParams)); } const requestCount = this.sessionSpawnRequests.length + this.sessionSendRequests.length + - this.githubPublishRequests.length; + this.githubPublishRequests.length + + this.portalRequests.length; if (requestCount <= (this.options.silenceSessionToolResponses ?? 0)) { return; } @@ -985,6 +1003,7 @@ describe("worker runtime", () => { "exec", "sessions_spawn", "sessions_send", + "portal", ]; await expect(runWorkerDescriptor(launch)).resolves.toMatchObject({ status: "completed" }); @@ -994,9 +1013,23 @@ describe("worker runtime", () => { "exec", "sessions_spawn", "sessions_send", + "portal", ]); }); + it("hides portal authority when the admitted Gateway lacks portal protocol support", async () => { + const { gateway, launch } = await setup(); + launch.assignment.toolAuthority.allowedToolNames = ["read", "portal"]; + launch.admission.handshake.protocolFeatures = + launch.admission.handshake.protocolFeatures.filter( + (feature) => feature !== WORKER_PORTAL_PROTOCOL_FEATURE, + ); + + await expect(runWorkerDescriptor(launch)).resolves.toMatchObject({ status: "completed" }); + + expect(gateway.inferenceRequests[0]?.context.tools?.map((tool) => tool.name)).toEqual(["read"]); + }); + it("runs with no tools when the Gateway authority is empty", async () => { const { gateway, launch } = await setup(); launch.assignment.toolAuthority.allowedToolNames = []; @@ -1142,6 +1175,28 @@ describe("worker runtime", () => { await connection.stop(); }); + it("never replays a portal operation after its response is lost", async () => { + const { gateway, launch } = await setup({ + heartbeatIntervalMs: 1, + ignoreHeartbeat: true, + silenceSessionToolResponses: 1, + }); + const connection = createWorkerConnection({ + endpoint: { kind: "unix", socketPath: gateway.socketPath }, + connectParams: buildWorkerConnectParams(launch), + requestTimeoutMs: 25, + reconnectBackoff: { initialMs: 1, maxMs: 1, factor: 1, jitter: 0 }, + }); + await connection.start(); + const request = { toolCallId: "call-portal-once", action: "open" as const, port: 3000 }; + + await expect(connection.requestPortal(request)).rejects.toMatchObject({ + name: "WorkerConnectionInterruptedError", + }); + expect(gateway.portalRequests).toEqual([request]); + await connection.stop(); + }); + it("fail-stops a stale mid-run transcript without duplicating or rebasing the paid tail", async () => { const { gateway, launch } = await setup({ transcriptFailureAtRequest: 2 }); diff --git a/src/worker/worker.runtime.ts b/src/worker/worker.runtime.ts index 03fe92ab6db6..48f9e43da73c 100644 --- a/src/worker/worker.runtime.ts +++ b/src/worker/worker.runtime.ts @@ -1,6 +1,10 @@ import { chmod, mkdtemp, realpath, rm, stat } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; +import { + WORKER_PORTAL_PROTOCOL_FEATURE, + type WorkerHelloOk, +} from "../../packages/gateway-protocol/src/schema/worker-admission.js"; import { isPathInside } from "../infra/path-guards.js"; import { registerSecretValueForRedaction } from "../logging/secret-redaction-registry.js"; import type { WorkerBrowserRuntime } from "./browser-runtime.js"; @@ -127,8 +131,9 @@ export async function runWorkerDescriptor( }); try { + let hello: WorkerHelloOk; try { - await connection.start(); + hello = await connection.start(); } catch (error) { const fenced = fencedResult(connection.state); if (fenced) { @@ -171,7 +176,10 @@ export async function runWorkerDescriptor( ? {} : { systemPrompt: descriptor.assignment.systemPrompt }), inferenceOptions: descriptor.assignment.inferenceOptions, - allowedToolNames: descriptor.assignment.toolAuthority.allowedToolNames, + allowedToolNames: descriptor.assignment.toolAuthority.allowedToolNames.filter( + (name) => + name !== "portal" || hello.protocolFeatures.includes(WORKER_PORTAL_PROTOCOL_FEATURE), + ), ...(descriptor.assignment.browser ? { browser: descriptor.assignment.browser } : {}), ...(options.browserRuntime ? { browserRuntime: options.browserRuntime } : {}), inference: { stream },