From 2278ca6952ea8e5f97decf932fde4ee2c7799e1f Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 13 Aug 2026 03:07:11 -0700 Subject: [PATCH] feat(ui): click the desktop to take control (#123096) * feat(ui): click the desktop to take control View-only desktop connections now escalate to control by clicking anywhere on the desktop stage (transparent overlay button, keyboard accessible, same 'Take control' accessible name). The toolbar button is removed; the connecting status overlay becomes click-through so control can still be requested mid-connect. * fix(ui): use cursor-action token for desktop take-control overlay * test(ui): prove take-control click above a real noVNC surface In-page RFB 3.8 fake server (security None) lets the production DesktopClient drive the real noVNC client; the overlay click must hit-test above the mounted canvas and reconnect with control. * fix(ui): narrow fake RFB socket payload to ArrayBuffer-backed bytes --- docs/gateway/cloud-workers.md | 2 +- .../desktop/desktop-panel-launcher-styles.ts | 2 + .../desktop/desktop-panel-styles.ts | 13 ++ ui/src/components/desktop/desktop-panel.ts | 22 ++- ui/src/e2e/desktop-panel.e2e.test.ts | 158 +++++++++++++++++- 5 files changed, 183 insertions(+), 14 deletions(-) diff --git a/docs/gateway/cloud-workers.md b/docs/gateway/cloud-workers.md index 0248983c7ea7..c337565ea5c1 100644 --- a/docs/gateway/cloud-workers.md +++ b/docs/gateway/cloud-workers.md @@ -204,7 +204,7 @@ Set `"desktop": true` in a crabbox profile's `settings` to lease worker boxes wi Operators with `operator.admin` access watch and control the desktop from the Control UI **Desktop** panel (also in the command palette). The panel lists desktop-capable environments from `environments.list`, shows only apps the provider advertised, and launches those apps through `worker.desktop.launch`. It connects through the Gateway, which forwards the box's loopback VNC over the same pinned SSH transport used for worker traffic — the desktop is never exposed on the box's network, and the VNC password is delivered only inside the authenticated `worker.desktop.observe` RPC result, never stored by the Gateway. -Connections start view-only. **Take control** requests an input-capable connection; only one controller is active at a time, and taking control disconnects the previous controller (they are downgraded to view-only). Up to 8 observers can watch one environment. The desktop forward starts on first observe and shuts down about a minute after the last observer disconnects; stopping or reclaiming the environment tears it down immediately. +Connections start view-only. Clicking the desktop takes control (an input-capable connection); only one controller is active at a time, and taking control disconnects the previous controller (they are downgraded to view-only). Up to 8 observers can watch one environment. The desktop forward starts on first observe and shuts down about a minute after the last observer disconnects; stopping or reclaiming the environment tears it down immediately. The Browser launcher starts one visible Chrome or Chromium process on the worker display with raw CDP bound to `127.0.0.1` and a fresh lease-scoped user-data directory. It does not import cookies, attach the Chrome extension relay, or use Chrome MCP. The operator toolbar and cloud-worker agent share this process. A worker turn receives the normal `browser` tool only when the lease advertises Browser, the bundled Browser plugin is active, and normal tool policy allows `browser`; workers without that capability keep the existing coding-tool catalog. Generic desktop `computer` control is not part of this Labs surface. diff --git a/ui/src/components/desktop/desktop-panel-launcher-styles.ts b/ui/src/components/desktop/desktop-panel-launcher-styles.ts index 582b7df9d173..fff7da3a30cc 100644 --- a/ui/src/components/desktop/desktop-panel-launcher-styles.ts +++ b/ui/src/components/desktop/desktop-panel-launcher-styles.ts @@ -55,6 +55,8 @@ export const desktopPanelLauncherStyles = css` .desktop-connecting { position: absolute; inset: 0; + /* Status overlay only; clicks must reach the take-control surface below. */ + pointer-events: none; display: flex; align-items: center; justify-content: center; diff --git a/ui/src/components/desktop/desktop-panel-styles.ts b/ui/src/components/desktop/desktop-panel-styles.ts index 46b9cd12d39b..d777c84fdb73 100644 --- a/ui/src/components/desktop/desktop-panel-styles.ts +++ b/ui/src/components/desktop/desktop-panel-styles.ts @@ -161,4 +161,17 @@ export const desktopPanelStyles = css` inset: 0; background: var(--bg); } + /* View-only affordance: clicking anywhere on the desktop takes control. */ + .desktop-stage__take-control { + position: absolute; + inset: 0; + border: 0; + padding: 0; + background: transparent; + cursor: var(--cursor-action, pointer); + } + .desktop-stage__take-control:focus-visible { + outline: 2px solid var(--accent, #ff5c5c); + outline-offset: -2px; + } `; diff --git a/ui/src/components/desktop/desktop-panel.ts b/ui/src/components/desktop/desktop-panel.ts index 4f07c8a11260..9708ebe44682 100644 --- a/ui/src/components/desktop/desktop-panel.ts +++ b/ui/src/components/desktop/desktop-panel.ts @@ -596,18 +596,6 @@ class OpenClawDesktopPanel extends OpenClawLitElement { ` : nothing} - ${!this.controlling - ? html`` - : nothing} ` + : nothing} ${this.state === "connecting" ? html`
diff --git a/ui/src/e2e/desktop-panel.e2e.test.ts b/ui/src/e2e/desktop-panel.e2e.test.ts index d09d8b7ae896..e27ee0ec3b4d 100644 --- a/ui/src/e2e/desktop-panel.e2e.test.ts +++ b/ui/src/e2e/desktop-panel.e2e.test.ts @@ -668,7 +668,24 @@ suite.define(() => { await panel.getByRole("button", { name: "Connect", exact: true }).click(); await expect.poll(async () => (await gateway.getRequests("desktop.observe")).length).toBe(2); - await panel.getByRole("button", { name: "Take control", exact: true }).click(); + const takeControl = panel.getByRole("button", { name: "Take control", exact: true }); + const overlayCoversStage = await panel.evaluate((element) => { + const stage = element.shadowRoot?.querySelector(".desktop-stage"); + const overlay = element.shadowRoot?.querySelector( + ".desktop-stage__take-control", + ); + if (!stage || !overlay) { + return false; + } + const stageRect = stage.getBoundingClientRect(); + const overlayRect = overlay.getBoundingClientRect(); + return ( + Math.abs(stageRect.width - overlayRect.width) < 1 && + Math.abs(stageRect.height - overlayRect.height) < 1 + ); + }); + expect(overlayCoversStage).toBe(true); + await takeControl.click(); await expect.poll(async () => (await gateway.getRequests("desktop.observe")).length).toBe(3); const observeRequests = await gateway.getRequests("desktop.observe"); expect(observeRequests[2]?.params).toEqual({ @@ -685,6 +702,145 @@ suite.define(() => { }); }); + it("takes control by clicking a real noVNC-mounted desktop", async () => { + await suite.withPage({ serviceWorkers: "block" }, async ({ page }) => { + const gateway = await installMockGateway(page, { + featureMethods: ["desktop.observe", "environments.list"], + methodResponses: { + "sessions.list": sessionsList("active"), + "environments.list": { environments: [workerDesktopEnvironment] }, + "desktop.observe": { + cases: [ + { + match: { + source: { kind: "environment", environmentId: "worker-desktop-1" }, + control: false, + }, + response: { + transport: "rfb", + wsPath: "/desktop/observe?token=view", + expiresAtMs: 60_000, + control: false, + }, + }, + { + match: { + source: { kind: "environment", environmentId: "worker-desktop-1" }, + control: true, + }, + response: { + transport: "rfb", + wsPath: "/desktop/observe?token=control", + expiresAtMs: 60_000, + control: true, + }, + }, + ], + }, + }, + }); + await page.goto(`${suite.server.baseUrl}chat`); + // Route desktop observe sockets to an in-page RFB 3.8 server (security + // None) so the production DesktopClient drives the real noVNC client. + await page.evaluate(() => { + const GatewaySocket = window.WebSocket; + class FakeRfbSocket extends EventTarget { + binaryType = "arraybuffer"; + protocol = ""; + readyState = 0; + onerror: ((event: Event) => void) | null = null; + onmessage: ((event: MessageEvent) => void) | null = null; + onopen: ((event: Event) => void) | null = null; + onclose: ((event: CloseEvent) => void) | null = null; + private handshake = 0; + constructor() { + super(); + setTimeout(() => { + if (this.readyState !== 0) { + return; + } + this.readyState = 1; + this.onopen?.(new Event("open")); + this.deliver(new TextEncoder().encode("RFB 003.008\n")); + }, 0); + } + private deliver(bytes: Uint8Array): void { + setTimeout(() => { + if (this.readyState === 1) { + this.onmessage?.(new MessageEvent("message", { data: bytes.buffer })); + } + }, 0); + } + send(): void { + // Handshake replies are fixed-size, so respond by stage instead of + // parsing: version -> security types, choice -> ok, init -> ServerInit. + this.handshake += 1; + if (this.handshake === 1) { + this.deliver(new Uint8Array([1, 1])); + } else if (this.handshake === 2) { + this.deliver(new Uint8Array([0, 0, 0, 0])); + } else if (this.handshake === 3) { + const name = new TextEncoder().encode("fake-desktop"); + const init = new Uint8Array(24 + name.length); + const view = new DataView(init.buffer); + view.setUint16(0, 800); + view.setUint16(2, 600); + init.set([32, 24, 0, 1], 4); + view.setUint16(8, 255); + view.setUint16(10, 255); + view.setUint16(12, 255); + init.set([16, 8, 0], 14); + view.setUint32(20, name.length); + init.set(name, 24); + this.deliver(init); + } + } + close(code = 1000, reason = ""): void { + if (this.readyState === 3) { + return; + } + this.readyState = 3; + const event = new CloseEvent("close", { code, reason }); + this.onclose?.(event); + this.dispatchEvent(new CloseEvent("close", { code, reason })); + } + } + const RoutedSocket = function (url: string, protocols?: string | string[]) { + return url.includes("/desktop/observe") + ? new FakeRfbSocket() + : new GatewaySocket(url, protocols); + }; + RoutedSocket.prototype = GatewaySocket.prototype; + Object.assign(RoutedSocket, { + CONNECTING: 0, + OPEN: 1, + CLOSING: 2, + CLOSED: 3, + }); + window.WebSocket = RoutedSocket as unknown as typeof WebSocket; + }); + + await openDirectDesktop(page, "worker-desktop-1"); + const panel = page.locator("openclaw-desktop-panel"); + await panel.locator("section[aria-label='Desktop']").waitFor(); + await panel.locator(".desktop-surface canvas").waitFor(); + + const takeControl = panel.getByRole("button", { name: "Take control", exact: true }); + await takeControl.waitFor(); + // Playwright refuses the click if the noVNC canvas intercepted it, so a + // successful click proves the overlay hit-tests above the real surface. + await takeControl.click(); + await expect.poll(async () => (await gateway.getRequests("desktop.observe")).length).toBe(2); + const observeRequests = await gateway.getRequests("desktop.observe"); + expect(observeRequests[1]?.params).toEqual({ + source: { kind: "environment", environmentId: "worker-desktop-1" }, + control: true, + }); + await panel.locator(".desktop-surface canvas").waitFor(); + expect(await takeControl.count()).toBe(0); + }); + }); + it("shows only apps advertised by the selected environment", async () => { await suite.withPage({ serviceWorkers: "block" }, async ({ page }) => { const gateway = await installMockGateway(page, {