diff --git a/ui/src/app/app-host.dock-suppression.test.ts b/ui/src/app/app-host.dock-suppression.test.ts index f225722147a4..b0b71df0c45a 100644 --- a/ui/src/app/app-host.dock-suppression.test.ts +++ b/ui/src/app/app-host.dock-suppression.test.ts @@ -23,7 +23,7 @@ afterEach(() => { }); describe("OpenClaw shell dock suppression", () => { - it("applies route and session ownership to shell panels", () => { + it("applies route ownership to shell panels without session-gating desktop", () => { vi.stubGlobal("localStorage", createStorageMock()); vi.stubGlobal( "matchMedia", @@ -41,12 +41,7 @@ describe("OpenClaw shell dock suppression", () => { hello: { auth: { role: "operator", scopes: ["operator.admin"] }, features: { - methods: [ - "terminal.open", - "browser.request", - "openclaw.chat", - "worker.desktop.observe", - ], + methods: ["terminal.open", "browser.request", "openclaw.chat", "desktop.observe"], }, }, lastError: null, @@ -157,7 +152,7 @@ describe("OpenClaw shell dock suppression", () => { } ).suppressed, ).toBe(false); - expect(desktopAvailable()).toBe(false); + expect(desktopAvailable()).toBe(true); context.sessions.state.result!.sessions = [ { @@ -174,10 +169,10 @@ describe("OpenClaw shell dock suppression", () => { { key: "agent:main:main", kind: "direct", updatedAt: 0 }, ]; renderLit(shell.render(), container); - expect(desktopAvailable()).toBe(false); + expect(desktopAvailable()).toBe(true); context.sessions.state.result = null; renderLit(shell.render(), container); - expect(desktopAvailable()).toBe(false); + expect(desktopAvailable()).toBe(true); }); }); diff --git a/ui/src/app/app-host.ts b/ui/src/app/app-host.ts index fce48efb0114..31ee44cfb48f 100644 --- a/ui/src/app/app-host.ts +++ b/ui/src/app/app-host.ts @@ -28,7 +28,6 @@ import { isGatewayMethodAdvertised } from "../lib/gateway-methods.ts"; import { createIdleImport } from "../lib/idle-import.ts"; import { isWorkboardEnabledInConfigSnapshot } from "../lib/plugin-activation.ts"; import { resolveSessionDisplayName } from "../lib/session-display.ts"; -import { findUiSessionRow } from "../lib/sessions/route-navigation.ts"; import { isUiGlobalSessionKey, normalizeAgentId, @@ -524,8 +523,7 @@ class OpenClawShell } const gatewaySnapshot = context.gateway?.snapshot; if (gatewaySnapshot) { - const activeSessionRow = findUiSessionRow(context, this.activeSessionKey); - const desktopAvailable = isDesktopPanelAvailable(gatewaySnapshot, activeSessionRow); + const desktopAvailable = isDesktopPanelAvailable(gatewaySnapshot); if (this.commandPalette) { this.commandPalette.desktopAvailable = desktopAvailable; } diff --git a/ui/src/app/app-shell-chrome.ts b/ui/src/app/app-shell-chrome.ts index cc04df3ab051..d8980a7a553c 100644 --- a/ui/src/app/app-shell-chrome.ts +++ b/ui/src/app/app-shell-chrome.ts @@ -1,5 +1,3 @@ -import { isCloudWorkerPlacementState } from "../../../packages/gateway-protocol/src/schema/session-placement-state.js"; -import type { GatewaySessionRow } from "../api/types.ts"; import { isSettingsNavigationRoute } from "../app-navigation.ts"; import { routeIdFromPath, type RouteId } from "../app-route-paths.ts"; import { @@ -24,7 +22,6 @@ import type { BoardFace } from "../lib/board/settings.ts"; import { isGatewayMethodAdvertised } from "../lib/gateway-methods.ts"; import { resolveAsciiShortcutKey } from "../lib/keyboard-shortcuts.ts"; import { readSessionMethodAccess } from "../lib/session-method-access.ts"; -import { findUiSessionRow } from "../lib/sessions/route-navigation.ts"; import { isTerminalAvailable } from "../lib/terminal-availability.ts"; import type { ShellRouteState } from "./app-host-route-state.ts"; import type { ApplicationContext, ApplicationNavigationOptions } from "./context.ts"; @@ -58,13 +55,11 @@ export function isBrowserPanelAvailable( export function isDesktopPanelAvailable( snapshot: ApplicationContext["gateway"]["snapshot"], - session: GatewaySessionRow | undefined, ): boolean { return ( - isCloudWorkerPlacementState(session?.placement?.state) && snapshot.phase === "connected" && hasOperatorAdminAccess(snapshot.hello?.auth ?? null) && - isGatewayMethodAdvertised(snapshot, "worker.desktop.observe") === true + isGatewayMethodAdvertised(snapshot, "desktop.observe") === true ); } @@ -508,8 +503,7 @@ export class ShellChromeOwner { readonly handleDeferredDesktopToggle = (event: Event): void => { const host = this.host; const context = host.context; - const session = context ? findUiSessionRow(context, host.activeSessionKey) : undefined; - if (!context || !isDesktopPanelAvailable(context.gateway.snapshot, session)) { + if (!context || !isDesktopPanelAvailable(context.gateway.snapshot)) { event.stopImmediatePropagation(); return; } diff --git a/ui/src/app/app-shell-view.ts b/ui/src/app/app-shell-view.ts index 4ccee15ef81e..065485e98a58 100644 --- a/ui/src/app/app-shell-view.ts +++ b/ui/src/app/app-shell-view.ts @@ -12,7 +12,6 @@ import type { ThemeModeChangeDetail } from "../components/theme-mode-toggle.ts"; import { t } from "../i18n/index.ts"; import { canCallGatewayMethod, isGatewayMethodAdvertised } from "../lib/gateway-methods.ts"; import { readSessionMethodAccess } from "../lib/session-method-access.ts"; -import { findUiSessionRow } from "../lib/sessions/route-navigation.ts"; import { normalizeAgentId } from "../lib/sessions/session-key.ts"; import { isTerminalAvailable } from "../lib/terminal-availability.ts"; import { findSettingsSearchBlocks } from "../pages/config/settings-search.ts"; @@ -152,8 +151,7 @@ export function renderApplicationShell(host: ShellViewHost) { context.config.current.terminalEnabled ?? false, ); const browserPanelAvailable = isBrowserPanelAvailable(gatewaySnapshot); - const activeSessionRow = findUiSessionRow(context, host.activeSessionKey); - const desktopPanelAvailable = isDesktopPanelAvailable(gatewaySnapshot, activeSessionRow); + const desktopPanelAvailable = isDesktopPanelAvailable(gatewaySnapshot); const custodianPanelAvailable = gatewayConnected && isGatewayMethodAdvertised(gatewaySnapshot, "openclaw.chat") === true; const activeRoute = host.routeState.routeId ?? "chat"; diff --git a/ui/src/components/desktop/desktop-client.test.ts b/ui/src/components/desktop/desktop-client.test.ts index 535dac9a444b..bc0e4a81a547 100644 --- a/ui/src/components/desktop/desktop-client.test.ts +++ b/ui/src/components/desktop/desktop-client.test.ts @@ -26,7 +26,7 @@ function createFakeRfb() { constructor( readonly target: HTMLElement, readonly channel: string | WebSocket, - readonly options?: { credentials?: { password: string } }, + readonly options?: { credentials?: { username?: string; password?: string } }, ) { super(); instances.push(this); @@ -52,7 +52,7 @@ describe("DesktopClient", () => { await client.connect({ gatewayUrl, wsUrl: "/desktop/observe?token=abc", - password: "secret", + credentials: { password: "secret" }, viewOnly: true, target, }); @@ -70,7 +70,7 @@ describe("DesktopClient", () => { const handle = await client.connect({ gatewayUrl: "ws://control.example.test", wsUrl: "/desktop/observe", - password: "secret", + credentials: { username: "operator", password: "secret" }, background: "rgb(8, 8, 8)", viewOnly: false, target: document.createElement("div"), @@ -79,7 +79,9 @@ describe("DesktopClient", () => { expect(instances[0]?.background).toBe("rgb(8, 8, 8)"); expect(instances[0]?.viewOnly).toBe(false); expect(instances[0]?.scaleViewport).toBe(true); - expect(instances[0]?.options).toEqual({ credentials: { password: "secret" } }); + expect(instances[0]?.options).toEqual({ + credentials: { username: "operator", password: "secret" }, + }); handle.disconnect(); expect(instances[0]?.disconnect).toHaveBeenCalledOnce(); diff --git a/ui/src/components/desktop/desktop-client.ts b/ui/src/components/desktop/desktop-client.ts index 25d77127b51b..8a21f324ff82 100644 --- a/ui/src/components/desktop/desktop-client.ts +++ b/ui/src/components/desktop/desktop-client.ts @@ -10,11 +10,11 @@ type DesktopSecurityFailureDetail = { type DesktopConnectOptions = { background?: string; + credentials?: { username?: string; password?: string }; gatewayUrl?: string; onConnect?: () => void; onDisconnect?: (detail: DesktopDisconnectDetail) => void; onSecurityFailure?: (detail: DesktopSecurityFailureDetail) => void; - password?: string; target: HTMLElement; viewOnly: boolean; wsUrl: string; @@ -34,7 +34,7 @@ type RfbClient = EventTarget & { type RfbConstructor = new ( target: HTMLElement, channel: string | WebSocket, - options?: { credentials?: { password: string } }, + options?: { credentials?: { username?: string; password?: string } }, ) => RfbClient; type RfbLoader = () => Promise; @@ -85,7 +85,7 @@ export class DesktopClient { const rfb = new Rfb( options.target, socket, - options.password ? { credentials: { password: options.password } } : undefined, + options.credentials ? { credentials: options.credentials } : undefined, ); rfb.background = options.background ?? getComputedStyle(options.target).backgroundColor; rfb.viewOnly = options.viewOnly; diff --git a/ui/src/components/desktop/desktop-panel.ts b/ui/src/components/desktop/desktop-panel.ts index f254366785d9..9545ee1d07e2 100644 --- a/ui/src/components/desktop/desktop-panel.ts +++ b/ui/src/components/desktop/desktop-panel.ts @@ -1,9 +1,10 @@ import type { + DesktopObserveResult, + DesktopSource, EnvironmentSummary, EnvironmentsListResult, WorkerDesktopAppId, WorkerDesktopLaunchResult, - WorkerDesktopObserveResult, } from "@openclaw/gateway-protocol"; import { css, html, nothing, svg } from "lit"; import { property, state } from "lit/decorators.js"; @@ -36,10 +37,22 @@ const panelLayout = createDockPanelLayout({ defaultWidth: 560, }); -type DesktopPanelState = "picker" | "connecting" | "connected" | "disconnected"; +type DesktopPanelState = "picker" | "credentials" | "connecting" | "connected" | "disconnected"; type DesktopAppId = WorkerDesktopAppId; +type DesktopCredentials = { username?: string; password?: string }; +type PendingDesktopConnection = { + environmentId: string; + observed: DesktopObserveResult; + operationId: number; +}; -/** `` — dockable RFB access to cloud-worker desktops. */ +function desktopSourceForEnvironment(environment: Pick): DesktopSource { + return environment.id === "gateway" + ? { kind: "host" } + : { kind: "environment", environmentId: environment.id }; +} + +/** `` — dockable RFB access to Gateway desktop sources. */ class OpenClawDesktopPanel extends OpenClawLitElement { @property({ attribute: false }) client: GatewayBrowserClient | null = null; @property({ type: Boolean }) available = false; @@ -52,6 +65,7 @@ class OpenClawDesktopPanel extends OpenClawLitElement { @state() private loading = false; @state() private state: DesktopPanelState = "picker"; @state() private environmentId: string | null = null; + @state() private source: DesktopSource | null = null; @state() private controlling = false; @state() private errorText: string | null = null; @state() private noticeText: string | null = null; @@ -61,6 +75,8 @@ class OpenClawDesktopPanel extends OpenClawLitElement { @state() private desktopApps: DesktopAppId[] = []; private connection: DesktopConnectionHandle | null = null; + private credentials: DesktopCredentials | undefined; + private pendingConnection: PendingDesktopConnection | null = null; private operationId = 0; private launchOperationId = 0; private controlTakeoverRecoveryUsed = false; @@ -171,6 +187,28 @@ class OpenClawDesktopPanel extends OpenClawLitElement { text-align: center; color: var(--muted, #8a919e); } + .desktop-credentials { + display: flex; + width: min(320px, 100%); + flex-direction: column; + gap: 10px; + text-align: left; + } + .desktop-credentials__label { + display: flex; + flex-direction: column; + gap: 5px; + color: var(--text, #d7dae0); + font-size: 12px; + } + .desktop-credentials__input { + border: 1px solid var(--border, #262b34); + border-radius: 6px; + padding: 7px 9px; + background: var(--bg, #111318); + color: var(--text, #d7dae0); + font: inherit; + } .desktop-environment { display: flex; align-items: center; @@ -230,6 +268,7 @@ class OpenClawDesktopPanel extends OpenClawLitElement { override disconnectedCallback(): void { window.removeEventListener(DESKTOP_PANEL_TOGGLE_EVENT, this.onToggleRequest); this.disconnectConnection(); + this.credentials = undefined; super.disconnectedCallback(); } @@ -289,6 +328,8 @@ class OpenClawDesktopPanel extends OpenClawLitElement { this.clearLaunchState(); this.state = "picker"; this.environmentId = null; + this.source = null; + this.credentials = undefined; this.desktopApps = []; this.controlling = false; this.disconnectedReason = null; @@ -296,6 +337,7 @@ class OpenClawDesktopPanel extends OpenClawLitElement { private disconnectConnection(): void { this.operationId += 1; + this.pendingConnection = null; const connection = this.connection; this.connection = null; connection?.disconnect(); @@ -320,9 +362,7 @@ class OpenClawDesktopPanel extends OpenClawLitElement { if (operationId !== this.operationId) { return; } - this.environments = result.environments.filter( - (environment) => environment.worker?.desktop === true, - ); + this.environments = result.environments.filter((environment) => environment.desktop === true); } catch (error) { if (operationId === this.operationId) { this.errorText = t("desktop.errors.listFailed", { error: formatUiError(error) }); @@ -345,6 +385,7 @@ class OpenClawDesktopPanel extends OpenClawLitElement { } if (this.environmentId !== environmentId) { this.clearLaunchState(); + this.credentials = undefined; this.desktopApps = [ ...(this.environments.find((environment) => environment.id === environmentId)?.worker ?.desktopApps ?? []), @@ -352,7 +393,12 @@ class OpenClawDesktopPanel extends OpenClawLitElement { } this.disconnectConnection(); const operationId = this.operationId; + const environment = this.environments.find((candidate) => candidate.id === environmentId) ?? { + id: environmentId, + }; + const source = desktopSourceForEnvironment(environment); this.environmentId = environmentId; + this.source = source; this.controlling = control; this.state = "connecting"; this.errorText = null; @@ -362,13 +408,39 @@ class OpenClawDesktopPanel extends OpenClawLitElement { } this.controlTakeoverRecoveryUsed = options.takeoverRecovery === true; try { - const observed = await client.request("worker.desktop.observe", { - environmentId, + const observed = await client.request("desktop.observe", { + source, control, }); if (operationId !== this.operationId) { return; } + const credentials = observed.vncPassword + ? { password: observed.vncPassword } + : observed.auth === "vnc-password" + ? this.credentials + : undefined; + if (observed.auth === "vnc-password" && !credentials?.password) { + this.pendingConnection = { environmentId, observed, operationId }; + this.state = "credentials"; + return; + } + await this.connectObserved({ environmentId, observed, operationId }, credentials); + } catch (error) { + this.failConnection(operationId, error); + } + } + + private async connectObserved( + pending: PendingDesktopConnection, + credentials?: DesktopCredentials, + ): Promise { + const client = this.client; + if (!client || pending.operationId !== this.operationId) { + return; + } + this.state = "connecting"; + try { await this.updateComplete; const target = this.shadowRoot?.querySelector(".desktop-surface"); if (!target) { @@ -378,43 +450,64 @@ class OpenClawDesktopPanel extends OpenClawLitElement { const background = getComputedStyle(target).backgroundColor; const connection = await desktopClient.connect({ background, - wsUrl: observed.wsPath, + wsUrl: pending.observed.wsPath, gatewayUrl: client.gatewayUrl, - password: observed.vncPassword, - viewOnly: !observed.control, + credentials, + viewOnly: !pending.observed.control, target, onConnect: () => { - if (operationId === this.operationId) { + if (pending.operationId === this.operationId) { this.state = "connected"; } }, onDisconnect: (detail) => { - if (operationId === this.operationId) { - this.handleDesktopDisconnect(environmentId, detail.code, detail.reason); + if (pending.operationId === this.operationId) { + this.handleDesktopDisconnect(pending.environmentId, detail.code, detail.reason); } }, onSecurityFailure: (detail) => { - if (operationId === this.operationId) { + if (pending.operationId === this.operationId) { this.errorText = t("desktop.errors.securityFailed", { reason: detail.reason ?? t("desktop.unknownReason"), }); } }, }); - if (operationId !== this.operationId) { + if (pending.operationId !== this.operationId) { connection.disconnect(); return; } this.connection = connection; } catch (error) { - if (operationId === this.operationId) { - this.state = "disconnected"; - this.disconnectedReason = formatUiError(error); - this.clearLaunchState(); - } + this.failConnection(pending.operationId, error); } } + private failConnection(operationId: number, error: unknown): void { + if (operationId !== this.operationId) { + return; + } + this.state = "disconnected"; + this.disconnectedReason = formatUiError(error); + this.clearLaunchState(); + } + + private handleCredentialsSubmit(event: SubmitEvent): void { + event.preventDefault(); + const pending = this.pendingConnection; + if (!pending || pending.operationId !== this.operationId) { + return; + } + const password = new FormData(event.currentTarget as HTMLFormElement).get("password"); + if (typeof password !== "string" || password.length === 0) { + return; + } + const credentials = { password }; + this.credentials = credentials; + this.pendingConnection = null; + void this.connectObserved(pending, credentials); + } + private handleDesktopDisconnect(environmentId: string, code?: number, reason?: string): void { this.connection = null; this.clearLaunchState(); @@ -438,10 +531,10 @@ class OpenClawDesktopPanel extends OpenClawLitElement { private async launchApp(app: DesktopAppId): Promise { const client = this.client; - const environmentId = this.environmentId; + const source = this.source; if ( !client || - !environmentId || + source?.kind !== "environment" || (this.state !== "connecting" && this.state !== "connected") || !this.desktopApps.includes(app) || this.launchingApp === app @@ -452,16 +545,16 @@ class OpenClawDesktopPanel extends OpenClawLitElement { this.launchingApp = app; this.launchErrorText = null; try { - await client.request("worker.desktop.launch", { - environmentId, + await client.request("desktop.launch", { + source, app, }); - if (operationId !== this.launchOperationId || environmentId !== this.environmentId) { + if (operationId !== this.launchOperationId || source !== this.source) { return; } this.launchingApp = null; } catch (error) { - if (operationId !== this.launchOperationId || environmentId !== this.environmentId) { + if (operationId !== this.launchOperationId || source !== this.source) { return; } this.launchingApp = null; @@ -533,10 +626,13 @@ class OpenClawDesktopPanel extends OpenClawLitElement { private renderEnvironment(environment: EnvironmentSummary) { const worker = environment.worker; + const source = desktopSourceForEnvironment(environment); return html`
-
${environment.id}
+
+ ${source.kind === "host" ? t("desktop.thisMachine") : environment.id} +
${worker?.state ?? environment.status}
@@ -562,7 +658,7 @@ class OpenClawDesktopPanel extends OpenClawLitElement { private renderConnection() { return html`
- ${this.desktopApps.length > 0 + ${this.source?.kind === "environment" && this.desktopApps.length > 0 ? html`
${this.desktopApps.map((app) => { const launching = this.launchingApp === app; @@ -652,6 +748,29 @@ class OpenClawDesktopPanel extends OpenClawLitElement { `; } + private renderCredentials() { + return html` +
+
+
${t("desktop.passwordPrompt")}
+ + +
+
+ `; + } + override render() { if (!this.available || !this.dockLayout.open) { return nothing; @@ -673,9 +792,11 @@ class OpenClawDesktopPanel extends OpenClawLitElement { : nothing} ${this.state === "picker" ? this.renderPicker() - : this.state === "disconnected" - ? this.renderDisconnected() - : this.renderConnection()} + : this.state === "credentials" + ? this.renderCredentials() + : this.state === "disconnected" + ? this.renderDisconnected() + : this.renderConnection()}
`; diff --git a/ui/src/e2e/desktop-panel.e2e.test.ts b/ui/src/e2e/desktop-panel.e2e.test.ts index d23c872b5fb4..39905d0f70e2 100644 --- a/ui/src/e2e/desktop-panel.e2e.test.ts +++ b/ui/src/e2e/desktop-panel.e2e.test.ts @@ -3,7 +3,7 @@ import { installMockGateway } from "../test-helpers/control-ui-e2e.ts"; import { createControlUiE2eSuite } from "./control-ui-e2e-suite.test-support.ts"; const suite = createControlUiE2eSuite({ - name: "cloud worker desktop panel", + name: "desktop source panel", startServerBeforeBrowser: true, unavailableMessage: (executablePath) => `Playwright Chromium is not installed or cannot start at ${executablePath}. Run \`pnpm --dir ui exec playwright install --with-deps chromium\`.`, @@ -48,11 +48,15 @@ async function installDesktopClientFake(panel: import("playwright").Locator) { ( element as HTMLElement & { desktopClientFactory: () => { - connect(): Promise<{ disconnect(): void }>; + connect(options: { credentials?: { password?: string } }): Promise<{ + disconnect(): void; + }>; }; } ).desktopClientFactory = () => ({ - async connect() { + async connect(options) { + element.dataset.connectCount = String(Number(element.dataset.connectCount ?? "0") + 1); + element.dataset.usedCredentials = options.credentials?.password ? "true" : "false"; return { disconnect() { element.dataset.disconnectCount = String( @@ -73,7 +77,7 @@ suite.define(() => { methodResponses: { "sessions.list": sessionsList("active") }, }, { - featureMethods: ["environments.list", "worker.desktop.observe"], + featureMethods: ["environments.list", "desktop.observe"], methodResponses: { "sessions.list": sessionsList("active") }, operatorScopes: ["operator.read"], }, @@ -87,34 +91,87 @@ suite.define(() => { } }); - it("keeps the desktop command and panel unavailable for a local session", async () => { + it("keeps the desktop command and panel available without a cloud session", async () => { await suite.withPage({ serviceWorkers: "block" }, async ({ page }) => { const gateway = await installMockGateway(page, { - featureMethods: ["environments.list", "worker.desktop.observe"], - methodResponses: { "sessions.list": sessionsList("local") }, + featureMethods: ["environments.list", "desktop.observe"], + methodResponses: { + "sessions.list": sessionsList("local"), + "environments.list": { environments: [] }, + }, }); await page.goto(`${suite.server.baseUrl}chat`); await openPalette(page); - expect(await page.getByRole("option", { name: "Desktop", exact: true }).count()).toBe(0); + expect(await page.getByRole("option", { name: "Desktop", exact: true }).count()).toBe(1); - await page.evaluate(() => { - window.dispatchEvent( - new CustomEvent("openclaw:desktop-toggle", { detail: { open: true } }), - ); + await page.getByRole("option", { name: "Desktop", exact: true }).click(); + await page.locator("openclaw-desktop-panel section[aria-label='Desktop']").waitFor(); + await gateway.waitForRequest("environments.list"); + }); + }); + + it("connects the host source after an in-memory VNC password prompt", async () => { + await suite.withPage({ serviceWorkers: "block" }, async ({ page }) => { + const gateway = await installMockGateway(page, { + featureMethods: ["desktop.observe", "environments.list"], + methodResponses: { + "sessions.list": sessionsList("local"), + "environments.list": { + environments: [ + { id: "gateway", type: "local", status: "available", desktop: true }, + { + id: "legacy-nested-worker", + type: "worker", + status: "available", + worker: { + providerId: "crabbox", + state: "ready", + ageMs: 1_000, + attachedSessionIds: [], + tunnelStatus: "connected", + desktop: true, + }, + }, + ], + }, + "desktop.observe": { + transport: "rfb", + wsPath: "/desktop/observe?token=host", + expiresAtMs: 60_000, + control: false, + auth: "vnc-password", + }, + }, }); - await page.waitForTimeout(250); - expect( - await page.locator("openclaw-desktop-panel section[aria-label='Desktop']").count(), - ).toBe(0); - expect(await gateway.getRequests("environments.list")).toHaveLength(0); + + const panel = await openDesktopPanel(page); + await gateway.waitForRequest("environments.list"); + await panel.getByText("This machine", { exact: true }).waitFor(); + expect(await panel.getByText("legacy-nested-worker", { exact: true }).count()).toBe(0); + await installDesktopClientFake(panel); + + await panel.getByRole("button", { name: "Connect", exact: true }).click(); + const observeRequest = await gateway.waitForRequest("desktop.observe"); + expect(observeRequest.params).toEqual({ source: { kind: "host" }, control: false }); + await panel.getByText("Enter the VNC password for this machine.", { exact: true }).waitFor(); + expect(await panel.getAttribute("data-connect-count")).toBeNull(); + + await panel.getByLabel("VNC password", { exact: true }).fill("memory-only-test-password"); + await panel.getByRole("button", { name: "Connect", exact: true }).click(); + await expect.poll(async () => await panel.getAttribute("data-connect-count")).toBe("1"); + expect(await panel.getAttribute("data-used-credentials")).toBe("true"); + expect(await panel.getByRole("button", { name: "Browser", exact: true }).count()).toBe(0); + expect(await panel.getByRole("button", { name: "Terminal", exact: true }).count()).toBe(0); + expect(await gateway.getRequests("desktop.observe")).toHaveLength(1); + expect(await gateway.getRequests("desktop.launch")).toHaveLength(0); }); }); it("launches advertised desktop apps and keeps observe controls working", async () => { await suite.withPage({ serviceWorkers: "block" }, async ({ page }) => { const gateway = await installMockGateway(page, { - deferredMethods: ["worker.desktop.launch"], - featureMethods: ["environments.list", "worker.desktop.launch", "worker.desktop.observe"], + deferredMethods: ["desktop.launch"], + featureMethods: ["desktop.launch", "desktop.observe", "environments.list"], methodResponses: { "sessions.list": sessionsList("active"), "environments.list": { @@ -123,22 +180,25 @@ suite.define(() => { id: "worker-desktop-1", type: "worker", status: "available", + desktop: true, worker: { providerId: "crabbox", state: "attached", ageMs: 1_000, attachedSessionIds: ["agent:main:desktop"], tunnelStatus: "connected", - desktop: true, desktopApps: ["browser", "terminal"], }, }, ], }, - "worker.desktop.observe": { + "desktop.observe": { cases: [ { - match: { environmentId: "worker-desktop-1", control: false }, + match: { + source: { kind: "environment", environmentId: "worker-desktop-1" }, + control: false, + }, response: { transport: "rfb", wsPath: "/desktop/observe?token=view", @@ -147,7 +207,10 @@ suite.define(() => { }, }, { - match: { environmentId: "worker-desktop-1", control: true }, + match: { + source: { kind: "environment", environmentId: "worker-desktop-1" }, + control: true, + }, response: { transport: "rfb", wsPath: "/desktop/observe?token=control", @@ -157,7 +220,7 @@ suite.define(() => { }, ], }, - "worker.desktop.launch": { app: "browser", status: "ready" }, + "desktop.launch": { app: "browser", status: "ready" }, }, }); @@ -168,8 +231,11 @@ suite.define(() => { await installDesktopClientFake(panel); await panel.getByRole("button", { name: "Connect", exact: true }).click(); - const viewRequest = await gateway.waitForRequest("worker.desktop.observe"); - expect(viewRequest.params).toEqual({ environmentId: "worker-desktop-1", control: false }); + const viewRequest = await gateway.waitForRequest("desktop.observe"); + expect(viewRequest.params).toEqual({ + source: { kind: "environment", environmentId: "worker-desktop-1" }, + control: false, + }); await panel.getByText("Connecting to desktop…", { exact: true }).waitFor(); await panel.getByRole("button", { name: "Browser", exact: true }).waitFor(); await panel.getByRole("button", { name: "Terminal", exact: true }).waitFor(); @@ -197,17 +263,20 @@ suite.define(() => { expect(stageUsesAppBackground).toBe(true); await browserButton.click(); - const launchRequest = await gateway.waitForRequest("worker.desktop.launch"); - expect(launchRequest.params).toEqual({ environmentId: "worker-desktop-1", app: "browser" }); + const launchRequest = await gateway.waitForRequest("desktop.launch"); + expect(launchRequest.params).toEqual({ + source: { kind: "environment", environmentId: "worker-desktop-1" }, + app: "browser", + }); await expect.poll(async () => await browserButton.getAttribute("aria-busy")).toBe("true"); expect(await terminalButton.isEnabled()).toBe(true); - await gateway.resolveDeferred("worker.desktop.launch", { app: "browser", status: "ready" }); + await gateway.resolveDeferred("desktop.launch", { app: "browser", status: "ready" }); await expect.poll(async () => await browserButton.getAttribute("aria-busy")).toBe("false"); - await gateway.deferNext("worker.desktop.launch"); + await gateway.deferNext("desktop.launch"); await browserButton.click(); - await gateway.waitForRequest("worker.desktop.launch"); - await gateway.rejectDeferred("worker.desktop.launch", { + await gateway.waitForRequest("desktop.launch"); + await gateway.rejectDeferred("desktop.launch", { message: "worker desktop app launch unavailable; try again", }); await panel @@ -218,24 +287,20 @@ suite.define(() => { expect(await browserButton.isEnabled()).toBe(true); await panel.getByRole("button", { name: "Disconnect", exact: true }).click(); - await panel.getByText("Cloud worker desktops", { exact: true }).waitFor(); + await panel.getByText("Desktop sources", { exact: true }).waitFor(); expect( await panel .getByText("worker desktop app launch unavailable; try again", { exact: true }) .count(), ).toBe(0); await panel.getByRole("button", { name: "Connect", exact: true }).click(); - await expect - .poll(async () => (await gateway.getRequests("worker.desktop.observe")).length) - .toBe(2); + await expect.poll(async () => (await gateway.getRequests("desktop.observe")).length).toBe(2); await panel.getByRole("button", { name: "Take control", exact: true }).click(); - await expect - .poll(async () => (await gateway.getRequests("worker.desktop.observe")).length) - .toBe(3); - const observeRequests = await gateway.getRequests("worker.desktop.observe"); + await expect.poll(async () => (await gateway.getRequests("desktop.observe")).length).toBe(3); + const observeRequests = await gateway.getRequests("desktop.observe"); expect(observeRequests[2]?.params).toEqual({ - environmentId: "worker-desktop-1", + source: { kind: "environment", environmentId: "worker-desktop-1" }, control: true, }); expect(await panel.getByRole("button", { name: "Take control", exact: true }).count()).toBe( @@ -243,7 +308,7 @@ suite.define(() => { ); await panel.getByRole("button", { name: "Disconnect", exact: true }).click(); - await panel.getByText("Cloud worker desktops", { exact: true }).waitFor(); + await panel.getByText("Desktop sources", { exact: true }).waitFor(); expect(Number((await panel.getAttribute("data-disconnect-count")) ?? "0")).toBeGreaterThan(0); }); }); @@ -251,7 +316,7 @@ suite.define(() => { it("shows only apps advertised by the selected environment", async () => { await suite.withPage({ serviceWorkers: "block" }, async ({ page }) => { const gateway = await installMockGateway(page, { - featureMethods: ["environments.list", "worker.desktop.launch", "worker.desktop.observe"], + featureMethods: ["desktop.launch", "desktop.observe", "environments.list"], methodResponses: { "sessions.list": sessionsList("active"), "environments.list": { @@ -260,19 +325,19 @@ suite.define(() => { id: "terminal-only-worker", type: "worker", status: "available", + desktop: true, worker: { providerId: "crabbox", state: "ready", ageMs: 1_000, attachedSessionIds: [], tunnelStatus: "connected", - desktop: true, desktopApps: ["terminal"], }, }, ], }, - "worker.desktop.observe": { + "desktop.observe": { transport: "rfb", wsPath: "/desktop/observe?token=view", expiresAtMs: 60_000, diff --git a/ui/src/i18n/locales/en.ts b/ui/src/i18n/locales/en.ts index fd6fc1e386ce..d7f7932ee4de 100644 --- a/ui/src/i18n/locales/en.ts +++ b/ui/src/i18n/locales/en.ts @@ -1972,23 +1972,25 @@ export const en: TranslationMap = { resize: "Resize desktop panel", dockBottom: "Dock to bottom", dockRight: "Dock to right", - pickerTitle: "Cloud worker desktops", + pickerTitle: "Desktop sources", + thisMachine: "This machine", refresh: "Refresh", refreshing: "Refreshing…", - loading: "Loading worker environments…", - empty: - "No desktop-capable worker environments exist. Enable one with desktop: true in a crabbox cloud-worker profile.", + loading: "Loading desktop sources…", + empty: "No desktop-capable sources are available.", connect: "Connect", connecting: "Connecting to desktop…", takeControl: "Take control", disconnect: "Disconnect", reconnect: "Reconnect", + passwordPrompt: "Enter the VNC password for this machine.", + passwordLabel: "VNC password", controlTaken: "Another operator took control", disconnected: "Desktop disconnected: {reason}", closeCode: "connection closed with code {code}", unknownReason: "unknown reason", errors: { - listFailed: "Could not load worker environments: {error}", + listFailed: "Could not load desktop sources: {error}", securityFailed: "Desktop security negotiation failed: {reason}", }, }, @@ -2857,6 +2859,11 @@ export const en: TranslationMap = { description: "Record content-free metadata for direct conversations in the audit ledger. Message content is never stored.", }, + hostDesktop: { + title: "Host Desktop", + description: + "Watch and control this Gateway machine from the Desktop panel through its existing VNC or Screen Sharing server.", + }, workerDesktop: { title: "Cloud Worker Desktop", description: diff --git a/ui/src/pages/chat/chat-pane-header.ts b/ui/src/pages/chat/chat-pane-header.ts index 9aa117a6e08a..104142305fc4 100644 --- a/ui/src/pages/chat/chat-pane-header.ts +++ b/ui/src/pages/chat/chat-pane-header.ts @@ -179,7 +179,7 @@ export abstract class ChatPaneHeader extends ChatPaneSessionMenu { session: row, }) : {}; - const desktopPanelAvailable = isDesktopPanelAvailable(this.context.gateway.snapshot, row); + const desktopPanelAvailable = isDesktopPanelAvailable(this.context.gateway.snapshot); const openDesktopPanel = () => window.dispatchEvent( new CustomEvent(DESKTOP_PANEL_TOGGLE_EVENT, { diff --git a/ui/src/pages/chat/chat-pane-terminal.test.ts b/ui/src/pages/chat/chat-pane-terminal.test.ts index 2efed0b432f9..67173ea42b10 100644 --- a/ui/src/pages/chat/chat-pane-terminal.test.ts +++ b/ui/src/pages/chat/chat-pane-terminal.test.ts @@ -66,7 +66,7 @@ describe("chat pane terminal action", () => { } }); - it("renders the desktop controls only for cloud sessions and opens the panel", () => { + it("renders desktop controls for local sessions when the source RPC is available", () => { const client = { request: vi.fn() } as unknown as GatewayBrowserClient; const { pane, state } = createTestChatPane({ client, sessions: {} as SessionCapability }); const localSession = { @@ -103,16 +103,16 @@ describe("chat pane terminal action", () => { renderHeader(cloudSession); expect(container.querySelector('[aria-label="Toggle desktop panel"]')).toBeNull(); - snapshot.hello = desktopHello(["worker.desktop.observe"], ["operator.admin"]); + snapshot.hello = desktopHello(["desktop.observe"], ["operator.admin"]); renderHeader(localSession); - expect(container.querySelector('[aria-label="Toggle desktop panel"]')).toBeNull(); - expect(panelActionIds()).not.toContain("desktop"); + expect(container.querySelector('[aria-label="Toggle desktop panel"]')).not.toBeNull(); + expect(panelActionIds()).toContain("desktop"); const events: CustomEvent[] = []; const listener = (event: Event) => events.push(event as CustomEvent); window.addEventListener(DESKTOP_PANEL_TOGGLE_EVENT, listener); try { - renderHeader(cloudSession); + renderHeader(localSession); const button = container.querySelector( '[aria-label="Toggle desktop panel"]', ); @@ -122,7 +122,7 @@ describe("chat pane terminal action", () => { expect(events).toHaveLength(1); expect(events[0]?.detail).toEqual({ open: true }); - snapshot.hello = desktopHello(["worker.desktop.observe"], ["operator.read"]); + snapshot.hello = desktopHello(["desktop.observe"], ["operator.read"]); renderHeader(cloudSession); expect(container.querySelector('[aria-label="Toggle desktop panel"]')).toBeNull(); } finally { diff --git a/ui/src/pages/labs/labs-page.test.ts b/ui/src/pages/labs/labs-page.test.ts index b0cdf324831a..0f18f9dd93d6 100644 --- a/ui/src/pages/labs/labs-page.test.ts +++ b/ui/src/pages/labs/labs-page.test.ts @@ -112,6 +112,7 @@ describe("LabsPage", () => { expect(page.querySelectorAll(".settings-row")).toHaveLength(LAB_FEATURES.length); expect(page.textContent).toContain("Code Mode"); expect(page.textContent).toContain("Swarm"); + expect(page.textContent).toContain("Host Desktop"); expect(page.textContent).toContain("Cloud Worker Desktop"); expect(codeModeToggle(page).checked).toBe(true); @@ -200,6 +201,12 @@ describe("LabsPage", () => { expectedPatch: { logging: { audit: { messages: "direct" } } }, note: "labs: update auditMessages", }, + { + label: "Host Desktop", + sourceConfig: { desktop: { host: { enabled: false } } }, + expectedPatch: { desktop: { host: { enabled: true } } }, + note: "labs: update hostDesktop", + }, { label: "Cloud Worker Desktop", sourceConfig: { cloudWorkers: { desktop: false } }, @@ -257,10 +264,11 @@ describe("LabsPage", () => { const rows = [...page.querySelectorAll(".settings-row")]; const restartRows = rows.filter((row) => row.textContent?.includes("restart")); - expect(restartRows).toHaveLength(2); + expect(restartRows).toHaveLength(3); expect(restartRows.map((row) => row.textContent)).toEqual( expect.arrayContaining([ expect.stringContaining("Message audit metadata"), + expect.stringContaining("Host Desktop"), expect.stringContaining("Cloud Worker Desktop"), ]), ); diff --git a/ui/src/pages/labs/labs-registry.ts b/ui/src/pages/labs/labs-registry.ts index 1944debce230..f8bc552d3cc5 100644 --- a/ui/src/pages/labs/labs-registry.ts +++ b/ui/src/pages/labs/labs-registry.ts @@ -194,6 +194,21 @@ export const LAB_FEATURES = [ // the recorder, so this outlives the reload plan's `logging: none` rule. restartHint: () => t("labsPage.restartRequired"), }, + { + id: "hostDesktop", + title: () => t("labsPage.hostDesktop.title"), + description: () => t("labsPage.hostDesktop.description"), + docsUrl: "https://docs.openclaw.ai/gateway/configuration-reference#desktop", + configPath: ["desktop", "host", "enabled"], + onValue: true, + offValue: false, + activeValues: [true], + readEnabled: null, + enableAlso: null, + resetScope: "gate", + // Method advertisement is resolved at Gateway startup, so the panel appears after restart. + restartHint: () => t("labsPage.restartRequired"), + }, { id: "workerDesktop", title: () => t("labsPage.workerDesktop.title"),