ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-23 02:45:38 -06:00
Code Issues Packages Projects Releases Wiki Activity

feat(ui): let the Desktop panel view this machine, not just cloud workers

Browse Source 
The Desktop panel was gated on a cloud-worker session placement, so an
operator running OpenClaw locally had no way to see the machine hosting
their main session even with a VNC server running on it.

Availability now follows the advertised desktop.observe method plus
operator.admin instead of session placement, and the picker lists every
environment whose summary reports a desktop, with the gateway row shown
as "This machine". Sources are passed to the generic desktop.observe /
desktop.launch RPCs; the app launcher stays worker-only. When a host
attach needs a password the gateway did not supply, the panel prompts and
keeps the value in memory for that connection only.

Adds the hostDesktop Labs toggle for desktop.host.enabled.
This commit is contained in:
Peter Steinberger
2026-08-11 23:14:06 -07:00
parent b2b27d3528
commit fc461f06ac
13 changed files with 324 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ui/src/app/app-host.dock-suppression.test.ts
+5 -10
View File
@@ -23,7 +23,7 @@ afterEach(() => {
});
describe("OpenClaw shell dock suppression", () => {
it("applies route and session ownership to shell panels", () => {
it("applies route ownership to shell panels without session-gating desktop", () => {
vi.stubGlobal("localStorage", createStorageMock());
vi.stubGlobal(
"matchMedia",
@@ -41,12 +41,7 @@ describe("OpenClaw shell dock suppression", () => {
hello: {
auth: { role: "operator", scopes: ["operator.admin"] },
features: {
methods: [
"terminal.open",
"browser.request",
"openclaw.chat",
"worker.desktop.observe",
],
methods: ["terminal.open", "browser.request", "openclaw.chat", "desktop.observe"],
},
},
lastError: null,
@@ -157,7 +152,7 @@ describe("OpenClaw shell dock suppression", () => {
}
).suppressed,
).toBe(false);
expect(desktopAvailable()).toBe(false);
expect(desktopAvailable()).toBe(true);
context.sessions.state.result!.sessions = [
{
@@ -174,10 +169,10 @@ describe("OpenClaw shell dock suppression", () => {
{ key: "agent:main:main", kind: "direct", updatedAt: 0 },
];
renderLit(shell.render(), container);
expect(desktopAvailable()).toBe(false);
expect(desktopAvailable()).toBe(true);
context.sessions.state.result = null;
renderLit(shell.render(), container);
expect(desktopAvailable()).toBe(false);
expect(desktopAvailable()).toBe(true);
});
});
ui/src/app/app-host.ts
+1 -3
View File
@@ -28,7 +28,6 @@ import { isGatewayMethodAdvertised } from "../lib/gateway-methods.ts";
import { createIdleImport } from "../lib/idle-import.ts";
import { isWorkboardEnabledInConfigSnapshot } from "../lib/plugin-activation.ts";
import { resolveSessionDisplayName } from "../lib/session-display.ts";
import { findUiSessionRow } from "../lib/sessions/route-navigation.ts";
import {
isUiGlobalSessionKey,
normalizeAgentId,
@@ -524,8 +523,7 @@ class OpenClawShell
}
const gatewaySnapshot = context.gateway?.snapshot;
if (gatewaySnapshot) {
const activeSessionRow = findUiSessionRow(context, this.activeSessionKey);
const desktopAvailable = isDesktopPanelAvailable(gatewaySnapshot, activeSessionRow);
const desktopAvailable = isDesktopPanelAvailable(gatewaySnapshot);
if (this.commandPalette) {
this.commandPalette.desktopAvailable = desktopAvailable;
}
ui/src/app/app-shell-chrome.ts
+2 -8
View File
@@ -1,5 +1,3 @@
import { isCloudWorkerPlacementState } from "../../../packages/gateway-protocol/src/schema/session-placement-state.js";
import type { GatewaySessionRow } from "../api/types.ts";
import { isSettingsNavigationRoute } from "../app-navigation.ts";
import { routeIdFromPath, type RouteId } from "../app-route-paths.ts";
import {
@@ -24,7 +22,6 @@ import type { BoardFace } from "../lib/board/settings.ts";
import { isGatewayMethodAdvertised } from "../lib/gateway-methods.ts";
import { resolveAsciiShortcutKey } from "../lib/keyboard-shortcuts.ts";
import { readSessionMethodAccess } from "../lib/session-method-access.ts";
import { findUiSessionRow } from "../lib/sessions/route-navigation.ts";
import { isTerminalAvailable } from "../lib/terminal-availability.ts";
import type { ShellRouteState } from "./app-host-route-state.ts";
import type { ApplicationContext, ApplicationNavigationOptions } from "./context.ts";
@@ -58,13 +55,11 @@ export function isBrowserPanelAvailable(
export function isDesktopPanelAvailable(
snapshot: ApplicationContext["gateway"]["snapshot"],
session: GatewaySessionRow | undefined,
): boolean {
return (
isCloudWorkerPlacementState(session?.placement?.state) &&
snapshot.phase === "connected" &&
hasOperatorAdminAccess(snapshot.hello?.auth ?? null) &&
isGatewayMethodAdvertised(snapshot, "worker.desktop.observe") === true
isGatewayMethodAdvertised(snapshot, "desktop.observe") === true
);
}
@@ -508,8 +503,7 @@ export class ShellChromeOwner {
readonly handleDeferredDesktopToggle = (event: Event): void => {
const host = this.host;
const context = host.context;
const session = context ? findUiSessionRow(context, host.activeSessionKey) : undefined;
if (!context || !isDesktopPanelAvailable(context.gateway.snapshot, session)) {
if (!context || !isDesktopPanelAvailable(context.gateway.snapshot)) {
event.stopImmediatePropagation();
return;
}
ui/src/app/app-shell-view.ts
+1 -3
View File
@@ -12,7 +12,6 @@ import type { ThemeModeChangeDetail } from "../components/theme-mode-toggle.ts";
import { t } from "../i18n/index.ts";
import { canCallGatewayMethod, isGatewayMethodAdvertised } from "../lib/gateway-methods.ts";
import { readSessionMethodAccess } from "../lib/session-method-access.ts";
import { findUiSessionRow } from "../lib/sessions/route-navigation.ts";
import { normalizeAgentId } from "../lib/sessions/session-key.ts";
import { isTerminalAvailable } from "../lib/terminal-availability.ts";
import { findSettingsSearchBlocks } from "../pages/config/settings-search.ts";
@@ -152,8 +151,7 @@ export function renderApplicationShell(host: ShellViewHost) {
context.config.current.terminalEnabled ?? false,
);
const browserPanelAvailable = isBrowserPanelAvailable(gatewaySnapshot);
const activeSessionRow = findUiSessionRow(context, host.activeSessionKey);
const desktopPanelAvailable = isDesktopPanelAvailable(gatewaySnapshot, activeSessionRow);
const desktopPanelAvailable = isDesktopPanelAvailable(gatewaySnapshot);
const custodianPanelAvailable =
gatewayConnected && isGatewayMethodAdvertised(gatewaySnapshot, "openclaw.chat") === true;
const activeRoute = host.routeState.routeId ?? "chat";
ui/src/components/desktop/desktop-client.test.ts
+6 -4
View File
@@ -26,7 +26,7 @@ function createFakeRfb() {
constructor(
readonly target: HTMLElement,
readonly channel: string | WebSocket,
readonly options?: { credentials?: { password: string } },
readonly options?: { credentials?: { username?: string; password?: string } },
) {
super();
instances.push(this);
@@ -52,7 +52,7 @@ describe("DesktopClient", () => {
await client.connect({
gatewayUrl,
wsUrl: "/desktop/observe?token=abc",
password: "secret",
credentials: { password: "secret" },
viewOnly: true,
target,
});
@@ -70,7 +70,7 @@ describe("DesktopClient", () => {
const handle = await client.connect({
gatewayUrl: "ws://control.example.test",
wsUrl: "/desktop/observe",
password: "secret",
credentials: { username: "operator", password: "secret" },
background: "rgb(8, 8, 8)",
viewOnly: false,
target: document.createElement("div"),
@@ -79,7 +79,9 @@ describe("DesktopClient", () => {
expect(instances[0]?.background).toBe("rgb(8, 8, 8)");
expect(instances[0]?.viewOnly).toBe(false);
expect(instances[0]?.scaleViewport).toBe(true);
expect(instances[0]?.options).toEqual({ credentials: { password: "secret" } });
expect(instances[0]?.options).toEqual({
credentials: { username: "operator", password: "secret" },
});
handle.disconnect();
expect(instances[0]?.disconnect).toHaveBeenCalledOnce();
ui/src/components/desktop/desktop-client.ts
+3 -3
View File
@@ -10,11 +10,11 @@ type DesktopSecurityFailureDetail = {
type DesktopConnectOptions = {
background?: string;
credentials?: { username?: string; password?: string };
gatewayUrl?: string;
onConnect?: () => void;
onDisconnect?: (detail: DesktopDisconnectDetail) => void;
onSecurityFailure?: (detail: DesktopSecurityFailureDetail) => void;
password?: string;
target: HTMLElement;
viewOnly: boolean;
wsUrl: string;
@@ -34,7 +34,7 @@ type RfbClient = EventTarget & {
type RfbConstructor = new (
target: HTMLElement,
channel: string | WebSocket,
options?: { credentials?: { password: string } },
options?: { credentials?: { username?: string; password?: string } },
) => RfbClient;
type RfbLoader = () => Promise<RfbConstructor>;
@@ -85,7 +85,7 @@ export class DesktopClient {
const rfb = new Rfb(
options.target,
socket,
options.password ? { credentials: { password: options.password } } : undefined,
options.credentials ? { credentials: options.credentials } : undefined,
);
rfb.background = options.background ?? getComputedStyle(options.target).backgroundColor;
rfb.viewOnly = options.viewOnly;
ui/src/components/desktop/desktop-panel.ts
+153 -32
View File
@@ -1,9 +1,10 @@
import type {
DesktopObserveResult,
DesktopSource,
EnvironmentSummary,
EnvironmentsListResult,
WorkerDesktopAppId,
WorkerDesktopLaunchResult,
WorkerDesktopObserveResult,
} from "@openclaw/gateway-protocol";
import { css, html, nothing, svg } from "lit";
import { property, state } from "lit/decorators.js";
@@ -36,10 +37,22 @@ const panelLayout = createDockPanelLayout({
defaultWidth: 560,
});
type DesktopPanelState = "picker" | "connecting" | "connected" | "disconnected";
type DesktopPanelState = "picker" | "credentials" | "connecting" | "connected" | "disconnected";
type DesktopAppId = WorkerDesktopAppId;
type DesktopCredentials = { username?: string; password?: string };
type PendingDesktopConnection = {
environmentId: string;
observed: DesktopObserveResult;
operationId: number;
};
/** `<openclaw-desktop-panel>` — dockable RFB access to cloud-worker desktops. */
function desktopSourceForEnvironment(environment: Pick<EnvironmentSummary, "id">): DesktopSource {
return environment.id === "gateway"
? { kind: "host" }
: { kind: "environment", environmentId: environment.id };
}
/** `<openclaw-desktop-panel>` — dockable RFB access to Gateway desktop sources. */
class OpenClawDesktopPanel extends OpenClawLitElement {
@property({ attribute: false }) client: GatewayBrowserClient | null = null;
@property({ type: Boolean }) available = false;
@@ -52,6 +65,7 @@ class OpenClawDesktopPanel extends OpenClawLitElement {
@state() private loading = false;
@state() private state: DesktopPanelState = "picker";
@state() private environmentId: string | null = null;
@state() private source: DesktopSource | null = null;
@state() private controlling = false;
@state() private errorText: string | null = null;
@state() private noticeText: string | null = null;
@@ -61,6 +75,8 @@ class OpenClawDesktopPanel extends OpenClawLitElement {
@state() private desktopApps: DesktopAppId[] = [];
private connection: DesktopConnectionHandle | null = null;
private credentials: DesktopCredentials | undefined;
private pendingConnection: PendingDesktopConnection | null = null;
private operationId = 0;
private launchOperationId = 0;
private controlTakeoverRecoveryUsed = false;
@@ -171,6 +187,28 @@ class OpenClawDesktopPanel extends OpenClawLitElement {
text-align: center;
color: var(--muted, #8a919e);
}
.desktop-credentials {
display: flex;
width: min(320px, 100%);
flex-direction: column;
gap: 10px;
text-align: left;
}
.desktop-credentials__label {
display: flex;
flex-direction: column;
gap: 5px;
color: var(--text, #d7dae0);
font-size: 12px;
}
.desktop-credentials__input {
border: 1px solid var(--border, #262b34);
border-radius: 6px;
padding: 7px 9px;
background: var(--bg, #111318);
color: var(--text, #d7dae0);
font: inherit;
}
.desktop-environment {
display: flex;
align-items: center;
@@ -230,6 +268,7 @@ class OpenClawDesktopPanel extends OpenClawLitElement {
override disconnectedCallback(): void {
window.removeEventListener(DESKTOP_PANEL_TOGGLE_EVENT, this.onToggleRequest);
this.disconnectConnection();
this.credentials = undefined;
super.disconnectedCallback();
}
@@ -289,6 +328,8 @@ class OpenClawDesktopPanel extends OpenClawLitElement {
this.clearLaunchState();
this.state = "picker";
this.environmentId = null;
this.source = null;
this.credentials = undefined;
this.desktopApps = [];
this.controlling = false;
this.disconnectedReason = null;
@@ -296,6 +337,7 @@ class OpenClawDesktopPanel extends OpenClawLitElement {
private disconnectConnection(): void {
this.operationId += 1;
this.pendingConnection = null;
const connection = this.connection;
this.connection = null;
connection?.disconnect();
@@ -320,9 +362,7 @@ class OpenClawDesktopPanel extends OpenClawLitElement {
if (operationId !== this.operationId) {
return;
}
this.environments = result.environments.filter(
(environment) => environment.worker?.desktop === true,
);
this.environments = result.environments.filter((environment) => environment.desktop === true);
} catch (error) {
if (operationId === this.operationId) {
this.errorText = t("desktop.errors.listFailed", { error: formatUiError(error) });
@@ -345,6 +385,7 @@ class OpenClawDesktopPanel extends OpenClawLitElement {
}
if (this.environmentId !== environmentId) {
this.clearLaunchState();
this.credentials = undefined;
this.desktopApps = [
...(this.environments.find((environment) => environment.id === environmentId)?.worker
?.desktopApps ?? []),
@@ -352,7 +393,12 @@ class OpenClawDesktopPanel extends OpenClawLitElement {
}
this.disconnectConnection();
const operationId = this.operationId;
const environment = this.environments.find((candidate) => candidate.id === environmentId) ?? {
id: environmentId,
};
const source = desktopSourceForEnvironment(environment);
this.environmentId = environmentId;
this.source = source;
this.controlling = control;
this.state = "connecting";
this.errorText = null;
@@ -362,13 +408,39 @@ class OpenClawDesktopPanel extends OpenClawLitElement {
}
this.controlTakeoverRecoveryUsed = options.takeoverRecovery === true;
try {
const observed = await client.request<WorkerDesktopObserveResult>("worker.desktop.observe", {
environmentId,
const observed = await client.request<DesktopObserveResult>("desktop.observe", {
source,
control,
});
if (operationId !== this.operationId) {
return;
}
const credentials = observed.vncPassword
? { password: observed.vncPassword }
: observed.auth === "vnc-password"
? this.credentials
: undefined;
if (observed.auth === "vnc-password" && !credentials?.password) {
this.pendingConnection = { environmentId, observed, operationId };
this.state = "credentials";
return;
}
await this.connectObserved({ environmentId, observed, operationId }, credentials);
} catch (error) {
this.failConnection(operationId, error);
}
}
private async connectObserved(
pending: PendingDesktopConnection,
credentials?: DesktopCredentials,
): Promise<void> {
const client = this.client;
if (!client || pending.operationId !== this.operationId) {
return;
}
this.state = "connecting";
try {
await this.updateComplete;
const target = this.shadowRoot?.querySelector<HTMLElement>(".desktop-surface");
if (!target) {
@@ -378,43 +450,64 @@ class OpenClawDesktopPanel extends OpenClawLitElement {
const background = getComputedStyle(target).backgroundColor;
const connection = await desktopClient.connect({
background,
wsUrl: observed.wsPath,
wsUrl: pending.observed.wsPath,
gatewayUrl: client.gatewayUrl,
password: observed.vncPassword,
viewOnly: !observed.control,
credentials,
viewOnly: !pending.observed.control,
target,
onConnect: () => {
if (operationId === this.operationId) {
if (pending.operationId === this.operationId) {
this.state = "connected";
}
},
onDisconnect: (detail) => {
if (operationId === this.operationId) {
this.handleDesktopDisconnect(environmentId, detail.code, detail.reason);
if (pending.operationId === this.operationId) {
this.handleDesktopDisconnect(pending.environmentId, detail.code, detail.reason);
}
},
onSecurityFailure: (detail) => {
if (operationId === this.operationId) {
if (pending.operationId === this.operationId) {
this.errorText = t("desktop.errors.securityFailed", {
reason: detail.reason ?? t("desktop.unknownReason"),
});
}
},
});
if (operationId !== this.operationId) {
if (pending.operationId !== this.operationId) {
connection.disconnect();
return;
}
this.connection = connection;
} catch (error) {
if (operationId === this.operationId) {
this.state = "disconnected";
this.disconnectedReason = formatUiError(error);
this.clearLaunchState();
}
this.failConnection(pending.operationId, error);
}
}
private failConnection(operationId: number, error: unknown): void {
if (operationId !== this.operationId) {
return;
}
this.state = "disconnected";
this.disconnectedReason = formatUiError(error);
this.clearLaunchState();
}
private handleCredentialsSubmit(event: SubmitEvent): void {
event.preventDefault();
const pending = this.pendingConnection;
if (!pending || pending.operationId !== this.operationId) {
return;
}
const password = new FormData(event.currentTarget as HTMLFormElement).get("password");
if (typeof password !== "string" || password.length === 0) {
return;
}
const credentials = { password };
this.credentials = credentials;
this.pendingConnection = null;
void this.connectObserved(pending, credentials);
}
private handleDesktopDisconnect(environmentId: string, code?: number, reason?: string): void {
this.connection = null;
this.clearLaunchState();
@@ -438,10 +531,10 @@ class OpenClawDesktopPanel extends OpenClawLitElement {
private async launchApp(app: DesktopAppId): Promise<void> {
const client = this.client;
const environmentId = this.environmentId;
const source = this.source;
if (
!client ||
!environmentId ||
source?.kind !== "environment" ||
(this.state !== "connecting" && this.state !== "connected") ||
!this.desktopApps.includes(app) ||
this.launchingApp === app
@@ -452,16 +545,16 @@ class OpenClawDesktopPanel extends OpenClawLitElement {
this.launchingApp = app;
this.launchErrorText = null;
try {
await client.request<WorkerDesktopLaunchResult>("worker.desktop.launch", {
environmentId,
await client.request<WorkerDesktopLaunchResult>("desktop.launch", {
source,
app,
});
if (operationId !== this.launchOperationId || environmentId !== this.environmentId) {
if (operationId !== this.launchOperationId || source !== this.source) {
return;
}
this.launchingApp = null;
} catch (error) {
if (operationId !== this.launchOperationId || environmentId !== this.environmentId) {
if (operationId !== this.launchOperationId || source !== this.source) {
return;
}
this.launchingApp = null;
@@ -533,10 +626,13 @@ class OpenClawDesktopPanel extends OpenClawLitElement {
private renderEnvironment(environment: EnvironmentSummary) {
const worker = environment.worker;
const source = desktopSourceForEnvironment(environment);
return html`
<div class="desktop-environment">
<div class="desktop-environment__details">
<div class="desktop-environment__id">${environment.id}</div>
<div class="desktop-environment__id">
${source.kind === "host" ? t("desktop.thisMachine") : environment.id}
</div>
<div class="desktop-environment__meta">
<span>${worker?.state ?? environment.status}</span>
</div>
@@ -562,7 +658,7 @@ class OpenClawDesktopPanel extends OpenClawLitElement {
private renderConnection() {
return html`
<div class="desktop-toolbar desktop-toolbar--connection">
${this.desktopApps.length > 0
${this.source?.kind === "environment" && this.desktopApps.length > 0
? html`<div class="desktop-apps">
${this.desktopApps.map((app) => {
const launching = this.launchingApp === app;
@@ -652,6 +748,29 @@ class OpenClawDesktopPanel extends OpenClawLitElement {
`;
}
private renderCredentials() {
return html`
<div class="desktop-status">
<form class="desktop-credentials" @submit=${this.handleCredentialsSubmit}>
<div>${t("desktop.passwordPrompt")}</div>
<label class="desktop-credentials__label">
${t("desktop.passwordLabel")}
<input
class="desktop-credentials__input"
name="password"
type="password"
autocomplete="off"
required
/>
</label>
<button class="desktop-button desktop-button--primary" type="submit">
${t("desktop.connect")}
</button>
</form>
</div>
`;
}
override render() {
if (!this.available || !this.dockLayout.open) {
return nothing;
@@ -673,9 +792,11 @@ class OpenClawDesktopPanel extends OpenClawLitElement {
: nothing}
${this.state === "picker"
? this.renderPicker()
: this.state === "disconnected"
? this.renderDisconnected()
: this.renderConnection()}
: this.state === "credentials"
? this.renderCredentials()
: this.state === "disconnected"
? this.renderDisconnected()
: this.renderConnection()}
</div>
</section>
`;
ui/src/e2e/desktop-panel.e2e.test.ts
+110 -45
View File
@@ -3,7 +3,7 @@ import { installMockGateway } from "../test-helpers/control-ui-e2e.ts";
import { createControlUiE2eSuite } from "./control-ui-e2e-suite.test-support.ts";
const suite = createControlUiE2eSuite({
name: "cloud worker desktop panel",
name: "desktop source panel",
startServerBeforeBrowser: true,
unavailableMessage: (executablePath) =>
`Playwright Chromium is not installed or cannot start at ${executablePath}. Run \`pnpm --dir ui exec playwright install --with-deps chromium\`.`,
@@ -48,11 +48,15 @@ async function installDesktopClientFake(panel: import("playwright").Locator) {
(
element as HTMLElement & {
desktopClientFactory: () => {
connect(): Promise<{ disconnect(): void }>;
connect(options: { credentials?: { password?: string } }): Promise<{
disconnect(): void;
}>;
};
}
).desktopClientFactory = () => ({
async connect() {
async connect(options) {
element.dataset.connectCount = String(Number(element.dataset.connectCount ?? "0") + 1);
element.dataset.usedCredentials = options.credentials?.password ? "true" : "false";
return {
disconnect() {
element.dataset.disconnectCount = String(
@@ -73,7 +77,7 @@ suite.define(() => {
methodResponses: { "sessions.list": sessionsList("active") },
},
{
featureMethods: ["environments.list", "worker.desktop.observe"],
featureMethods: ["environments.list", "desktop.observe"],
methodResponses: { "sessions.list": sessionsList("active") },
operatorScopes: ["operator.read"],
},
@@ -87,34 +91,87 @@ suite.define(() => {
}
});
it("keeps the desktop command and panel unavailable for a local session", async () => {
it("keeps the desktop command and panel available without a cloud session", async () => {
await suite.withPage({ serviceWorkers: "block" }, async ({ page }) => {
const gateway = await installMockGateway(page, {
featureMethods: ["environments.list", "worker.desktop.observe"],
methodResponses: { "sessions.list": sessionsList("local") },
featureMethods: ["environments.list", "desktop.observe"],
methodResponses: {
"sessions.list": sessionsList("local"),
"environments.list": { environments: [] },
},
});
await page.goto(`${suite.server.baseUrl}chat`);
await openPalette(page);
expect(await page.getByRole("option", { name: "Desktop", exact: true }).count()).toBe(0);
expect(await page.getByRole("option", { name: "Desktop", exact: true }).count()).toBe(1);
await page.evaluate(() => {
window.dispatchEvent(
new CustomEvent("openclaw:desktop-toggle", { detail: { open: true } }),
);
await page.getByRole("option", { name: "Desktop", exact: true }).click();
await page.locator("openclaw-desktop-panel section[aria-label='Desktop']").waitFor();
await gateway.waitForRequest("environments.list");
});
});
it("connects the host source after an in-memory VNC password prompt", async () => {
await suite.withPage({ serviceWorkers: "block" }, async ({ page }) => {
const gateway = await installMockGateway(page, {
featureMethods: ["desktop.observe", "environments.list"],
methodResponses: {
"sessions.list": sessionsList("local"),
"environments.list": {
environments: [
{ id: "gateway", type: "local", status: "available", desktop: true },
{
id: "legacy-nested-worker",
type: "worker",
status: "available",
worker: {
providerId: "crabbox",
state: "ready",
ageMs: 1_000,
attachedSessionIds: [],
tunnelStatus: "connected",
desktop: true,
},
},
],
},
"desktop.observe": {
transport: "rfb",
wsPath: "/desktop/observe?token=host",
expiresAtMs: 60_000,
control: false,
auth: "vnc-password",
},
},
});
await page.waitForTimeout(250);
expect(
await page.locator("openclaw-desktop-panel section[aria-label='Desktop']").count(),
).toBe(0);
expect(await gateway.getRequests("environments.list")).toHaveLength(0);
const panel = await openDesktopPanel(page);
await gateway.waitForRequest("environments.list");
await panel.getByText("This machine", { exact: true }).waitFor();
expect(await panel.getByText("legacy-nested-worker", { exact: true }).count()).toBe(0);
await installDesktopClientFake(panel);
await panel.getByRole("button", { name: "Connect", exact: true }).click();
const observeRequest = await gateway.waitForRequest("desktop.observe");
expect(observeRequest.params).toEqual({ source: { kind: "host" }, control: false });
await panel.getByText("Enter the VNC password for this machine.", { exact: true }).waitFor();
expect(await panel.getAttribute("data-connect-count")).toBeNull();
await panel.getByLabel("VNC password", { exact: true }).fill("memory-only-test-password");
await panel.getByRole("button", { name: "Connect", exact: true }).click();
await expect.poll(async () => await panel.getAttribute("data-connect-count")).toBe("1");
expect(await panel.getAttribute("data-used-credentials")).toBe("true");
expect(await panel.getByRole("button", { name: "Browser", exact: true }).count()).toBe(0);
expect(await panel.getByRole("button", { name: "Terminal", exact: true }).count()).toBe(0);
expect(await gateway.getRequests("desktop.observe")).toHaveLength(1);
expect(await gateway.getRequests("desktop.launch")).toHaveLength(0);
});
});
it("launches advertised desktop apps and keeps observe controls working", async () => {
await suite.withPage({ serviceWorkers: "block" }, async ({ page }) => {
const gateway = await installMockGateway(page, {
deferredMethods: ["worker.desktop.launch"],
featureMethods: ["environments.list", "worker.desktop.launch", "worker.desktop.observe"],
deferredMethods: ["desktop.launch"],
featureMethods: ["desktop.launch", "desktop.observe", "environments.list"],
methodResponses: {
"sessions.list": sessionsList("active"),
"environments.list": {
@@ -123,22 +180,25 @@ suite.define(() => {
id: "worker-desktop-1",
type: "worker",
status: "available",
desktop: true,
worker: {
providerId: "crabbox",
state: "attached",
ageMs: 1_000,
attachedSessionIds: ["agent:main:desktop"],
tunnelStatus: "connected",
desktop: true,
desktopApps: ["browser", "terminal"],
},
},
],
},
"worker.desktop.observe": {
"desktop.observe": {
cases: [
{
match: { environmentId: "worker-desktop-1", control: false },
match: {
source: { kind: "environment", environmentId: "worker-desktop-1" },
control: false,
},
response: {
transport: "rfb",
wsPath: "/desktop/observe?token=view",
@@ -147,7 +207,10 @@ suite.define(() => {
},
},
{
match: { environmentId: "worker-desktop-1", control: true },
match: {
source: { kind: "environment", environmentId: "worker-desktop-1" },
control: true,
},
response: {
transport: "rfb",
wsPath: "/desktop/observe?token=control",
@@ -157,7 +220,7 @@ suite.define(() => {
},
],
},
"worker.desktop.launch": { app: "browser", status: "ready" },
"desktop.launch": { app: "browser", status: "ready" },
},
});
@@ -168,8 +231,11 @@ suite.define(() => {
await installDesktopClientFake(panel);
await panel.getByRole("button", { name: "Connect", exact: true }).click();
const viewRequest = await gateway.waitForRequest("worker.desktop.observe");
expect(viewRequest.params).toEqual({ environmentId: "worker-desktop-1", control: false });
const viewRequest = await gateway.waitForRequest("desktop.observe");
expect(viewRequest.params).toEqual({
source: { kind: "environment", environmentId: "worker-desktop-1" },
control: false,
});
await panel.getByText("Connecting to desktop…", { exact: true }).waitFor();
await panel.getByRole("button", { name: "Browser", exact: true }).waitFor();
await panel.getByRole("button", { name: "Terminal", exact: true }).waitFor();
@@ -197,17 +263,20 @@ suite.define(() => {
expect(stageUsesAppBackground).toBe(true);
await browserButton.click();
const launchRequest = await gateway.waitForRequest("worker.desktop.launch");
expect(launchRequest.params).toEqual({ environmentId: "worker-desktop-1", app: "browser" });
const launchRequest = await gateway.waitForRequest("desktop.launch");
expect(launchRequest.params).toEqual({
source: { kind: "environment", environmentId: "worker-desktop-1" },
app: "browser",
});
await expect.poll(async () => await browserButton.getAttribute("aria-busy")).toBe("true");
expect(await terminalButton.isEnabled()).toBe(true);
await gateway.resolveDeferred("worker.desktop.launch", { app: "browser", status: "ready" });
await gateway.resolveDeferred("desktop.launch", { app: "browser", status: "ready" });
await expect.poll(async () => await browserButton.getAttribute("aria-busy")).toBe("false");
await gateway.deferNext("worker.desktop.launch");
await gateway.deferNext("desktop.launch");
await browserButton.click();
await gateway.waitForRequest("worker.desktop.launch");
await gateway.rejectDeferred("worker.desktop.launch", {
await gateway.waitForRequest("desktop.launch");
await gateway.rejectDeferred("desktop.launch", {
message: "worker desktop app launch unavailable; try again",
});
await panel
@@ -218,24 +287,20 @@ suite.define(() => {
expect(await browserButton.isEnabled()).toBe(true);
await panel.getByRole("button", { name: "Disconnect", exact: true }).click();
await panel.getByText("Cloud worker desktops", { exact: true }).waitFor();
await panel.getByText("Desktop sources", { exact: true }).waitFor();
expect(
await panel
.getByText("worker desktop app launch unavailable; try again", { exact: true })
.count(),
).toBe(0);
await panel.getByRole("button", { name: "Connect", exact: true }).click();
await expect
.poll(async () => (await gateway.getRequests("worker.desktop.observe")).length)
.toBe(2);
await expect.poll(async () => (await gateway.getRequests("desktop.observe")).length).toBe(2);
await panel.getByRole("button", { name: "Take control", exact: true }).click();
await expect
.poll(async () => (await gateway.getRequests("worker.desktop.observe")).length)
.toBe(3);
const observeRequests = await gateway.getRequests("worker.desktop.observe");
await expect.poll(async () => (await gateway.getRequests("desktop.observe")).length).toBe(3);
const observeRequests = await gateway.getRequests("desktop.observe");
expect(observeRequests[2]?.params).toEqual({
environmentId: "worker-desktop-1",
source: { kind: "environment", environmentId: "worker-desktop-1" },
control: true,
});
expect(await panel.getByRole("button", { name: "Take control", exact: true }).count()).toBe(
@@ -243,7 +308,7 @@ suite.define(() => {
);
await panel.getByRole("button", { name: "Disconnect", exact: true }).click();
await panel.getByText("Cloud worker desktops", { exact: true }).waitFor();
await panel.getByText("Desktop sources", { exact: true }).waitFor();
expect(Number((await panel.getAttribute("data-disconnect-count")) ?? "0")).toBeGreaterThan(0);
});
});
@@ -251,7 +316,7 @@ suite.define(() => {
it("shows only apps advertised by the selected environment", async () => {
await suite.withPage({ serviceWorkers: "block" }, async ({ page }) => {
const gateway = await installMockGateway(page, {
featureMethods: ["environments.list", "worker.desktop.launch", "worker.desktop.observe"],
featureMethods: ["desktop.launch", "desktop.observe", "environments.list"],
methodResponses: {
"sessions.list": sessionsList("active"),
"environments.list": {
@@ -260,19 +325,19 @@ suite.define(() => {
id: "terminal-only-worker",
type: "worker",
status: "available",
desktop: true,
worker: {
providerId: "crabbox",
state: "ready",
ageMs: 1_000,
attachedSessionIds: [],
tunnelStatus: "connected",
desktop: true,
desktopApps: ["terminal"],
},
},
],
},
"worker.desktop.observe": {
"desktop.observe": {
transport: "rfb",
wsPath: "/desktop/observe?token=view",
expiresAtMs: 60_000,
ui/src/i18n/locales/en.ts
+12 -5
View File
@@ -1972,23 +1972,25 @@ export const en: TranslationMap = {
resize: "Resize desktop panel",
dockBottom: "Dock to bottom",
dockRight: "Dock to right",
pickerTitle: "Cloud worker desktops",
pickerTitle: "Desktop sources",
thisMachine: "This machine",
refresh: "Refresh",
refreshing: "Refreshing…",
loading: "Loading worker environments…",
empty:
"No desktop-capable worker environments exist. Enable one with desktop: true in a crabbox cloud-worker profile.",
loading: "Loading desktop sources…",
empty: "No desktop-capable sources are available.",
connect: "Connect",
connecting: "Connecting to desktop…",
takeControl: "Take control",
disconnect: "Disconnect",
reconnect: "Reconnect",
passwordPrompt: "Enter the VNC password for this machine.",
passwordLabel: "VNC password",
controlTaken: "Another operator took control",
disconnected: "Desktop disconnected: {reason}",
closeCode: "connection closed with code {code}",
unknownReason: "unknown reason",
errors: {
listFailed: "Could not load worker environments: {error}",
listFailed: "Could not load desktop sources: {error}",
securityFailed: "Desktop security negotiation failed: {reason}",
},
},
@@ -2857,6 +2859,11 @@ export const en: TranslationMap = {
description:
"Record content-free metadata for direct conversations in the audit ledger. Message content is never stored.",
},
hostDesktop: {
title: "Host Desktop",
description:
"Watch and control this Gateway machine from the Desktop panel through its existing VNC or Screen Sharing server.",
},
workerDesktop: {
title: "Cloud Worker Desktop",
description:
ui/src/pages/chat/chat-pane-header.ts
+1 -1
View File
@@ -179,7 +179,7 @@ export abstract class ChatPaneHeader extends ChatPaneSessionMenu {
session: row,
})
: {};
const desktopPanelAvailable = isDesktopPanelAvailable(this.context.gateway.snapshot, row);
const desktopPanelAvailable = isDesktopPanelAvailable(this.context.gateway.snapshot);
const openDesktopPanel = () =>
window.dispatchEvent(
new CustomEvent<DesktopPanelToggleDetail>(DESKTOP_PANEL_TOGGLE_EVENT, {
ui/src/pages/chat/chat-pane-terminal.test.ts
+6 -6
View File
@@ -66,7 +66,7 @@ describe("chat pane terminal action", () => {
}
});
it("renders the desktop controls only for cloud sessions and opens the panel", () => {
it("renders desktop controls for local sessions when the source RPC is available", () => {
const client = { request: vi.fn() } as unknown as GatewayBrowserClient;
const { pane, state } = createTestChatPane({ client, sessions: {} as SessionCapability });
const localSession = {
@@ -103,16 +103,16 @@ describe("chat pane terminal action", () => {
renderHeader(cloudSession);
expect(container.querySelector('[aria-label="Toggle desktop panel"]')).toBeNull();
snapshot.hello = desktopHello(["worker.desktop.observe"], ["operator.admin"]);
snapshot.hello = desktopHello(["desktop.observe"], ["operator.admin"]);
renderHeader(localSession);
expect(container.querySelector('[aria-label="Toggle desktop panel"]')).toBeNull();
expect(panelActionIds()).not.toContain("desktop");
expect(container.querySelector('[aria-label="Toggle desktop panel"]')).not.toBeNull();
expect(panelActionIds()).toContain("desktop");
const events: CustomEvent<DesktopPanelToggleDetail>[] = [];
const listener = (event: Event) => events.push(event as CustomEvent<DesktopPanelToggleDetail>);
window.addEventListener(DESKTOP_PANEL_TOGGLE_EVENT, listener);
try {
renderHeader(cloudSession);
renderHeader(localSession);
const button = container.querySelector<HTMLButtonElement>(
'[aria-label="Toggle desktop panel"]',
);
@@ -122,7 +122,7 @@ describe("chat pane terminal action", () => {
expect(events).toHaveLength(1);
expect(events[0]?.detail).toEqual({ open: true });
snapshot.hello = desktopHello(["worker.desktop.observe"], ["operator.read"]);
snapshot.hello = desktopHello(["desktop.observe"], ["operator.read"]);
renderHeader(cloudSession);
expect(container.querySelector('[aria-label="Toggle desktop panel"]')).toBeNull();
} finally {
ui/src/pages/labs/labs-page.test.ts
+9 -1
View File
@@ -112,6 +112,7 @@ describe("LabsPage", () => {
expect(page.querySelectorAll(".settings-row")).toHaveLength(LAB_FEATURES.length);
expect(page.textContent).toContain("Code Mode");
expect(page.textContent).toContain("Swarm");
expect(page.textContent).toContain("Host Desktop");
expect(page.textContent).toContain("Cloud Worker Desktop");
expect(codeModeToggle(page).checked).toBe(true);
@@ -200,6 +201,12 @@ describe("LabsPage", () => {
expectedPatch: { logging: { audit: { messages: "direct" } } },
note: "labs: update auditMessages",
},
{
label: "Host Desktop",
sourceConfig: { desktop: { host: { enabled: false } } },
expectedPatch: { desktop: { host: { enabled: true } } },
note: "labs: update hostDesktop",
},
{
label: "Cloud Worker Desktop",
sourceConfig: { cloudWorkers: { desktop: false } },
@@ -257,10 +264,11 @@ describe("LabsPage", () => {
const rows = [...page.querySelectorAll(".settings-row")];
const restartRows = rows.filter((row) => row.textContent?.includes("restart"));
expect(restartRows).toHaveLength(2);
expect(restartRows).toHaveLength(3);
expect(restartRows.map((row) => row.textContent)).toEqual(
expect.arrayContaining([
expect.stringContaining("Message audit metadata"),
expect.stringContaining("Host Desktop"),
expect.stringContaining("Cloud Worker Desktop"),
]),
);
ui/src/pages/labs/labs-registry.ts
+15
View File
@@ -194,6 +194,21 @@ export const LAB_FEATURES = [
// the recorder, so this outlives the reload plan's `logging: none` rule.
restartHint: () => t("labsPage.restartRequired"),
},
{
id: "hostDesktop",
title: () => t("labsPage.hostDesktop.title"),
description: () => t("labsPage.hostDesktop.description"),
docsUrl: "https://docs.openclaw.ai/gateway/configuration-reference#desktop",
configPath: ["desktop", "host", "enabled"],
onValue: true,
offValue: false,
activeValues: [true],
readEnabled: null,
enableAlso: null,
resetScope: "gate",
// Method advertisement is resolved at Gateway startup, so the panel appears after restart.
restartHint: () => t("labsPage.restartRequired"),
},
{
id: "workerDesktop",
title: () => t("labsPage.workerDesktop.title"),