diff --git a/apps/android/app/src/main/java/ai/openclaw/app/ui/DesktopScreen.kt b/apps/android/app/src/main/java/ai/openclaw/app/ui/DesktopScreen.kt index 89b83d3d44a2..0662a71e9abb 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/ui/DesktopScreen.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/ui/DesktopScreen.kt @@ -34,6 +34,7 @@ import androidx.core.net.toUri internal fun DesktopScreen( viewModel: MainViewModel, source: String? = null, + session: String? = null, onBack: () -> Unit, ) { val isConnected by viewModel.isConnected.collectAsState() @@ -72,10 +73,10 @@ internal fun DesktopScreen( val page = controlPage if (isConnected && page != null) { // GatewayControlPage equality includes credentials and the accepted TLS pin. - key(page, source) { + key(page, source, session) { ControlUiWebView( page = page, - url = desktopUrl(baseUrl = page.baseUrl, source = source), + url = desktopUrl(baseUrl = page.baseUrl, source = source, session = session), modifier = Modifier.fillMaxSize(), ) } @@ -106,6 +107,7 @@ internal fun DesktopScreen( internal fun desktopUrl( baseUrl: String, source: String? = null, + session: String? = null, ): String { val baseUri = baseUrl.trimEnd('/').toUri() val routePath = "${baseUri.encodedPath.orEmpty().trimEnd('/')}/" @@ -117,5 +119,6 @@ internal fun desktopUrl( .fragment(null) .appendQueryParameter("view", "desktop") source?.let { builder.appendQueryParameter("source", it) } + session?.let { builder.appendQueryParameter("session", it) } return builder.build().toString() } diff --git a/apps/android/app/src/main/java/ai/openclaw/app/ui/SessionDashboardScreen.kt b/apps/android/app/src/main/java/ai/openclaw/app/ui/SessionDashboardScreen.kt index c172467ae480..aa9c5990031d 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/ui/SessionDashboardScreen.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/ui/SessionDashboardScreen.kt @@ -48,9 +48,7 @@ internal fun SessionDashboardScreen( // The viewer replaces this screen in place rather than pushing a shell tab, so it must // claim System Back itself; the shell handler would otherwise pop the whole dashboard. BackHandler { showingDesktop = false } - // Session summaries do not advertise an environment id, so the viewer opens - // its source picker instead of guessing a gateway or node association. - DesktopScreen(viewModel = viewModel, source = null, onBack = { showingDesktop = false }) + DesktopScreen(viewModel = viewModel, session = sessionKey, onBack = { showingDesktop = false }) return } ClawScaffold( diff --git a/apps/android/app/src/test/java/ai/openclaw/app/ui/DesktopScreenTest.kt b/apps/android/app/src/test/java/ai/openclaw/app/ui/DesktopScreenTest.kt index b76dd0e86403..799d66380c22 100644 --- a/apps/android/app/src/test/java/ai/openclaw/app/ui/DesktopScreenTest.kt +++ b/apps/android/app/src/test/java/ai/openclaw/app/ui/DesktopScreenTest.kt @@ -34,4 +34,33 @@ class DesktopScreenTest { assertFalse(url.contains("token=")) assertFalse(url.contains("password=")) } + + @Test + fun desktopUrlEncodesProvidedSession() { + val url = + desktopUrl( + baseUrl = "https://gateway.example.com:8443/openclaw/", + session = "agent:main:mobile session", + ) + + assertEquals( + "https://gateway.example.com:8443/openclaw/?view=desktop&session=agent%3Amain%3Amobile%20session", + url, + ) + } + + @Test + fun desktopUrlRetainsExplicitSourceAlongsideSession() { + val url = + desktopUrl( + baseUrl = "https://gateway.example.com:8443", + source = "node:worker-1", + session = "agent:main:mobile", + ) + + assertEquals( + "https://gateway.example.com:8443/?view=desktop&source=node%3Aworker-1&session=agent%3Amain%3Amobile", + url, + ) + } } diff --git a/apps/ios/Sources/Chat/SessionDashboardScreen.swift b/apps/ios/Sources/Chat/SessionDashboardScreen.swift index 1733508ce49d..4278d4e0dbed 100644 --- a/apps/ios/Sources/Chat/SessionDashboardScreen.swift +++ b/apps/ios/Sources/Chat/SessionDashboardScreen.swift @@ -54,9 +54,7 @@ struct SessionDashboardScreen: View { } } .navigationDestination(isPresented: self.$showsDesktop) { - // Session dashboard presentation currently carries only the session key, - // so the desktop document mode owns source selection. - DesktopHubScreen(source: nil, usesNativeNavigationChrome: true) + DesktopHubScreen(session: self.sessionKey, usesNativeNavigationChrome: true) } } diff --git a/apps/ios/Sources/Desktop/DesktopHubScreen.swift b/apps/ios/Sources/Desktop/DesktopHubScreen.swift index ac30c9f52d52..bb2eca1787d1 100644 --- a/apps/ios/Sources/Desktop/DesktopHubScreen.swift +++ b/apps/ios/Sources/Desktop/DesktopHubScreen.swift @@ -6,17 +6,20 @@ import SwiftUI struct DesktopHubScreen: View { @Environment(NodeAppModel.self) private var appModel let source: String? + let session: String? let headerSidebarAction: OpenClawSidebarHeaderAction? let usesNativeNavigationChrome: Bool let gatewayAction: (() -> Void)? init( source: String? = nil, + session: String? = nil, headerSidebarAction: OpenClawSidebarHeaderAction? = nil, usesNativeNavigationChrome: Bool = false, gatewayAction: (() -> Void)? = nil) { self.source = source + self.session = session self.headerSidebarAction = headerSidebarAction self.usesNativeNavigationChrome = usesNativeNavigationChrome self.gatewayAction = gatewayAction @@ -27,17 +30,19 @@ struct DesktopHubScreen: View { let storedOperatorToken = AuthenticatedControlUI.storedOperatorToken(config: config) ZStack { OpenClawProBackground() - if let url = Self.desktopURL(config: config, source: self.source) { + if let url = Self.desktopURL(config: config, source: self.source, session: self.session) { AuthenticatedControlUIWebView( url: url, authScript: Self.desktopAuthUserScript( config: config, source: self.source, + session: self.session, storedOperatorToken: storedOperatorToken), tls: config?.tls) .id(Self.webContentIdentity( config: config, source: self.source, + session: self.session, storedOperatorToken: storedOperatorToken)) .ignoresSafeArea(.container, edges: .bottom) } else { @@ -90,38 +95,52 @@ struct DesktopHubScreen: View { /// Credentials never enter this URL; the document-start user script carries /// them through the Control UI's native-auth contract. - static func desktopURL(config: GatewayConnectConfig?, source: String?) -> URL? { + static func desktopURL( + config: GatewayConnectConfig?, + source: String?, + session: String? = nil) -> URL? + { var queryItems = [URLQueryItem(name: "view", value: "desktop")] if let source = self.normalizedSource(source) { queryItems.append(URLQueryItem(name: "source", value: source)) } + if let session = self.normalizedSource(session) { + queryItems.append(URLQueryItem(name: "session", value: session)) + } return AuthenticatedControlUI.pageURL( config: config, path: "/", queryItems: queryItems) } - static func desktopAuthUserScript(config: GatewayConnectConfig?, source: String?) -> String? { + static func desktopAuthUserScript( + config: GatewayConnectConfig?, + source: String?, + session: String? = nil) -> String? + { self.desktopAuthUserScript( config: config, source: source, + session: session, storedOperatorToken: AuthenticatedControlUI.storedOperatorToken(config: config)) } static func desktopAuthUserScript( config: GatewayConnectConfig?, source: String?, + session: String? = nil, storedOperatorToken: String?) -> String? { AuthenticatedControlUI.authUserScript( config: config, - pageURL: self.desktopURL(config: config, source: source), + pageURL: self.desktopURL(config: config, source: source, session: session), storedOperatorToken: storedOperatorToken) } static func webContentIdentity( config: GatewayConnectConfig?, source: String?, + session: String? = nil, storedOperatorToken: String?) -> Int { var hasher = Hasher() @@ -129,6 +148,7 @@ struct DesktopHubScreen: View { config: config, storedOperatorToken: storedOperatorToken)) hasher.combine(self.normalizedSource(source)) + hasher.combine(self.normalizedSource(session)) return hasher.finalize() } diff --git a/apps/ios/Tests/DesktopHubScreenTests.swift b/apps/ios/Tests/DesktopHubScreenTests.swift index cdb3358889f9..9dfde13862e0 100644 --- a/apps/ios/Tests/DesktopHubScreenTests.swift +++ b/apps/ios/Tests/DesktopHubScreenTests.swift @@ -34,28 +34,42 @@ struct DesktopHubScreenTests { token: "secret-token", password: "secret-password") - let url = DesktopHubScreen.desktopURL(config: config, source: nil) + let url = DesktopHubScreen.desktopURL(config: config, source: nil, session: nil) #expect(url?.absoluteString == "https://gateway.example.com:8443/openclaw/?view=desktop") #expect(url?.absoluteString.contains("secret-token") == false) #expect(url?.absoluteString.contains("secret-password") == false) } - @Test func `session desktop URL includes the selected source`() throws { + @Test func `session desktop URL includes the session key`() throws { let config = try Self.makeConfig( url: #require(URL(string: "ws://192.168.1.10:18789")), token: "secret-token") - let url = DesktopHubScreen.desktopURL(config: config, source: "node:worker-1") + let url = DesktopHubScreen.desktopURL( + config: config, + source: nil, + session: "agent:main:mobile session") - #expect(url?.absoluteString == "http://192.168.1.10:18789/?view=desktop&source=node%3Aworker-1") + #expect(url?.absoluteString == "http://192.168.1.10:18789/?view=desktop&session=agent%3Amain%3Amobile%20session") #expect(url?.absoluteString.contains("secret-token") == false) } + @Test func `explicit desktop source is retained alongside the session`() throws { + let config = try Self.makeConfig(url: #require(URL(string: "wss://gateway.example.com"))) + + let url = DesktopHubScreen.desktopURL( + config: config, + source: "node:worker-1", + session: "agent:main:mobile") + + #expect(url?.absoluteString == "https://gateway.example.com/?view=desktop&source=node%3Aworker-1&session=agent%3Amain%3Amobile") + } + @Test func `empty desktop source is omitted`() throws { let config = try Self.makeConfig(url: #require(URL(string: "wss://gateway.example.com"))) - let url = DesktopHubScreen.desktopURL(config: config, source: " ") + let url = DesktopHubScreen.desktopURL(config: config, source: " ", session: " ") #expect(url?.absoluteString == "https://gateway.example.com/?view=desktop") } diff --git a/apps/ios/UITests/OpenClawSnapshotUITests.swift b/apps/ios/UITests/OpenClawSnapshotUITests.swift index 25c485b77668..194f0908eb82 100644 --- a/apps/ios/UITests/OpenClawSnapshotUITests.swift +++ b/apps/ios/UITests/OpenClawSnapshotUITests.swift @@ -55,7 +55,7 @@ final class OpenClawSnapshotUITests: XCTestCase { let focusProbe = "focus" input.typeText(focusProbe) XCTAssertEqual(input.value as? String, focusProbe) - input.typeText(String(repeating: XCUIKeyboardKey.delete.rawValue, count: focusProbe.count)) + self.clearTextField(input) XCTAssertEqual(input.value as? String, "") app.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.2)).tap() if keyboard.exists { @@ -918,12 +918,14 @@ final class OpenClawSnapshotUITests: XCTestCase { let host = app.textFields["Host"] XCTAssertTrue(host.waitForExistence(timeout: 5)) host.tap() - host.typeText(String(repeating: XCUIKeyboardKey.delete.rawValue, count: 32) + "localhost") + self.clearTextField(host) + host.typeText("localhost") let port = app.textFields["Port"] XCTAssertTrue(port.waitForExistence(timeout: 5)) port.tap() - port.typeText(String(repeating: XCUIKeyboardKey.delete.rawValue, count: 5) + "18920") + self.clearTextField(port) + port.typeText("18920") let unencrypted = app.buttons["Unencrypted"] XCTAssertTrue(unencrypted.waitForExistence(timeout: 5)) unencrypted.tap() @@ -1523,6 +1525,19 @@ extension OpenClawSnapshotUITests { app.descendants(matching: .any)["chat-message-input"] } + /// A burst of synthetic key events can drop a keystroke under simulator load, so one + /// delete-per-character `typeText` does not reliably empty a field. Re-send against + /// whatever is actually left instead of assuming the first burst landed in full. + private func clearTextField(_ element: XCUIElement, attempts: Int = 4) { + for _ in 0 ..< attempts { + let value = element.value as? String ?? "" + if value.isEmpty { + return + } + element.typeText(String(repeating: XCUIKeyboardKey.delete.rawValue, count: value.count)) + } + } + private func attachFullScreenScreenshot(named name: String) { let attachment = XCTAttachment(screenshot: XCUIScreen.main.screenshot()) attachment.name = name diff --git a/ui/src/app/app-root.ts b/ui/src/app/app-root.ts index 1c5510afb8b9..f38d33949706 100644 --- a/ui/src/app/app-root.ts +++ b/ui/src/app/app-root.ts @@ -250,6 +250,7 @@ export class OpenClawApp extends OpenClawLightDomElement { .available=${desktopAvailable} .documentMode=${true} .documentSource=${this.desktopOptions.source} + .documentSession=${this.desktopOptions.session} .documentControl=${this.desktopOptions.control} .onDocumentClose=${() => { if (globalThis.history.length > 1) { diff --git a/ui/src/app/desktop-document-mode.test.ts b/ui/src/app/desktop-document-mode.test.ts new file mode 100644 index 000000000000..d663970c2292 --- /dev/null +++ b/ui/src/app/desktop-document-mode.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, it } from "vitest"; +import type { GatewaySessionRow } from "../api/types.ts"; +import { resolveDesktopDocumentTarget } from "../components/desktop/desktop-source.ts"; +import { desktopDocumentOptions } from "./desktop-document-mode.ts"; + +describe("desktop document mode", () => { + it("parses desktop source, session, and control options", () => { + expect( + desktopDocumentOptions({ + search: "?view=desktop&source=gateway&session=agent%3Amain%3Awork&control=1", + }), + ).toEqual({ + source: "gateway", + session: "agent:main:work", + control: true, + }); + }); + + it("prefers an explicit source over the session placement", () => { + const session = { + key: "agent:main:work", + kind: "direct", + updatedAt: 1, + execNode: "workstation", + } satisfies GatewaySessionRow; + + expect( + resolveDesktopDocumentTarget( + { source: "gateway", session: session.key, control: false }, + session, + ), + ).toBe("gateway"); + }); + + it.each([ + [ + "cloud placement", + { + key: "agent:main:cloud", + kind: "direct", + updatedAt: 1, + placement: { state: "active", environmentId: "worker:cloud-1" }, + } as GatewaySessionRow, + "worker:cloud-1", + ], + [ + "execution node", + { + key: "agent:main:node", + kind: "direct", + updatedAt: 1, + execNode: "workstation", + } satisfies GatewaySessionRow, + "node:workstation", + ], + [ + "gateway fallback", + { + key: "agent:main:gateway", + kind: "direct", + updatedAt: 1, + } satisfies GatewaySessionRow, + "gateway", + ], + ])("resolves a session's %s through the chat placement owner", (_label, session, expected) => { + expect( + resolveDesktopDocumentTarget({ source: null, session: session.key, control: false }, session), + ).toBe(expected); + }); + + it("returns no target for an unknown session", () => { + expect( + resolveDesktopDocumentTarget( + { source: null, session: "agent:main:missing", control: false }, + undefined, + ), + ).toBeNull(); + }); +}); diff --git a/ui/src/app/desktop-document-mode.ts b/ui/src/app/desktop-document-mode.ts index 98a61e43b4a3..9c816f3ebefb 100644 --- a/ui/src/app/desktop-document-mode.ts +++ b/ui/src/app/desktop-document-mode.ts @@ -4,8 +4,9 @@ const DESKTOP_DOCUMENT_PATH = "/desktop"; type DesktopDocumentLocation = Pick; -type DesktopDocumentOptions = { +export type DesktopDocumentOptions = { source: string | null; + session: string | null; control: boolean; }; @@ -33,6 +34,7 @@ export function desktopDocumentOptions( const search = new URLSearchParams(location?.search ?? ""); return { source: search.get("source"), + session: search.get("session"), control: search.get("control") === "1", }; } diff --git a/ui/src/components/desktop/desktop-mobile-keyboard.ts b/ui/src/components/desktop/desktop-mobile-keyboard.ts new file mode 100644 index 000000000000..97749d0064a9 --- /dev/null +++ b/ui/src/components/desktop/desktop-mobile-keyboard.ts @@ -0,0 +1,75 @@ +/** + * Mobile browsers only report composed text through an input's value, so the desktop document + * keeps a padded sentinel in the field and derives key events from how that value changed. + */ +const MOBILE_KEYBOARD_SENTINEL = "________________"; + +type MobileKeyboardOptions = { + /** Live RFB handle, or null while disconnected; absent senders mean a view-only transport. */ + connection: () => { + sendBackspace?: () => void; + sendKeyboardEvent?: (event: KeyboardEvent) => void; + sendText?: (text: string) => void; + } | null; + controlling: () => boolean; + input: () => HTMLTextAreaElement | null | undefined; +}; + +/** Bridges the desktop document's hidden textarea to the remote desktop's keyboard. */ +export class DesktopMobileKeyboard { + /** The document view renders this so the field always holds deletable padding. */ + value = MOBILE_KEYBOARD_SENTINEL; + + constructor(private readonly options: MobileKeyboardOptions) {} + + focus(): void { + const input = this.options.input(); + input?.focus({ preventScroll: true }); + input?.setSelectionRange(input.value.length, input.value.length); + } + + reset(input?: HTMLTextAreaElement): void { + this.value = MOBILE_KEYBOARD_SENTINEL; + const target = input ?? this.options.input(); + if (target) { + target.value = MOBILE_KEYBOARD_SENTINEL; + } + } + + handleKeyboardEvent(event: KeyboardEvent): void { + const connection = this.options.connection(); + if (!this.options.controlling() || !connection?.sendKeyboardEvent) { + return; + } + connection.sendKeyboardEvent(event); + event.preventDefault(); + } + + handleInput(event: InputEvent): void { + const input = event.currentTarget as HTMLTextAreaElement; + if (!this.options.controlling()) { + this.reset(input); + return; + } + const nextValue = input.value; + const connection = this.options.connection(); + let prefixLength = 0; + const comparableLength = Math.min(this.value.length, nextValue.length); + while ( + prefixLength < comparableLength && + this.value.charAt(prefixLength) === nextValue.charAt(prefixLength) + ) { + prefixLength += 1; + } + for (let index = this.value.length - prefixLength; index > 0; index -= 1) { + connection?.sendBackspace?.(); + } + connection?.sendText?.(nextValue.slice(prefixLength)); + // Refill once the field drifts outside the range that keeps further deletes reportable. + if (nextValue.length < 1 || nextValue.length > MOBILE_KEYBOARD_SENTINEL.length * 2) { + this.reset(input); + return; + } + this.value = nextValue; + } +} diff --git a/ui/src/components/desktop/desktop-panel.ts b/ui/src/components/desktop/desktop-panel.ts index 4bcbfbd79d27..4078cf3b25f0 100644 --- a/ui/src/components/desktop/desktop-panel.ts +++ b/ui/src/components/desktop/desktop-panel.ts @@ -8,6 +8,7 @@ import type { import { html, nothing } from "lit"; import { property, state } from "lit/decorators.js"; import type { GatewayBrowserClient } from "../../api/gateway.ts"; +import type { GatewaySessionRow } from "../../api/types.ts"; import { t } from "../../i18n/index.ts"; import { formatUiError } from "../../lib/format-error.ts"; import { OpenClawLitElement } from "../../lit/openclaw-element.ts"; @@ -20,6 +21,7 @@ import { import { DesktopClient, type DesktopConnectionHandle } from "./desktop-client.ts"; import { desktopDocumentStyles } from "./desktop-document-styles.ts"; import { renderDesktopDocumentView } from "./desktop-document-view.ts"; +import { DesktopMobileKeyboard } from "./desktop-mobile-keyboard.ts"; import type { DesktopAppId, DesktopCredentials, @@ -38,7 +40,7 @@ import { renderDesktopPanelHeader, renderDesktopPicker, } from "./desktop-panel-view.ts"; -import { desktopSourceForEnvironment } from "./desktop-source.ts"; +import { desktopSourceForEnvironment, resolveDesktopDocumentTarget } from "./desktop-source.ts"; const panelLayout = createDockPanelLayout({ storageKey: "openclaw.desktopPanel", @@ -49,7 +51,6 @@ const panelLayout = createDockPanelLayout({ defaultHeight: 420, defaultWidth: 560, }); -const MOBILE_KEYBOARD_SENTINEL = "________________"; /** `` — dockable RFB access to Gateway desktop sources. */ class OpenClawDesktopPanel extends OpenClawLitElement { @property({ attribute: false }) client: GatewayBrowserClient | null = null; @@ -57,6 +58,7 @@ class OpenClawDesktopPanel extends OpenClawLitElement { @property({ type: Boolean }) suppressed = false; @property({ type: Boolean }) documentMode = false; @property({ attribute: false }) documentSource: string | null = null; + @property({ attribute: false }) documentSession: string | null = null; @property({ type: Boolean }) documentControl = false; @property({ attribute: false }) onDocumentClose: (() => void) | null = null; @@ -85,7 +87,11 @@ class OpenClawDesktopPanel extends OpenClawLitElement { private launchOperationId = 0; private controlTakeoverRecoveryUsed = false; private documentSourceResolved = false; - private keyboardInputValue = MOBILE_KEYBOARD_SENTINEL; + private readonly mobileKeyboard = new DesktopMobileKeyboard({ + connection: () => this.connection, + controlling: () => this.controlling, + input: () => this.shadowRoot?.querySelector(".desktop-keyboard-input"), + }); private readonly dockLayout = new DockLayoutController(this, { layout: panelLayout, reservationPrefix: "desktop", @@ -132,13 +138,14 @@ class OpenClawDesktopPanel extends OpenClawLitElement { void this.refreshEnvironments(); } } - if (changed.has("documentSource")) { + if (changed.has("documentSource") || changed.has("documentSession")) { this.documentSourceResolved = false; } const gatewayAvailabilityChanged = changed.has("client") || changed.has("available"); const documentPresentationChanged = changed.has("documentMode") || changed.has("documentSource") || + changed.has("documentSession") || changed.has("documentControl"); if (this.documentMode && (gatewayAvailabilityChanged || documentPresentationChanged)) { if (!this.available) { @@ -211,7 +218,7 @@ class OpenClawDesktopPanel extends OpenClawLitElement { const connection = this.connection; this.connection = null; connection?.disconnect(); - this.resetDocumentKeyboardInput(); + this.mobileKeyboard.reset(); } private clearLaunchState(): void { @@ -239,7 +246,9 @@ class OpenClawDesktopPanel extends OpenClawLitElement { } catch (error) { if (operationId === this.operationId) { this.errorText = t("desktop.errors.listFailed", { error: formatUiError(error) }); - if (this.documentMode && this.documentSource !== null) { + if (this.documentMode && (this.documentSource !== null || this.documentSession !== null)) { + // A session key only names a machine once the inventory loads, so it stays out of + // `environmentId`; document-mode retry refreshes the inventory rather than reconnecting. this.environmentId = this.documentSource; this.state = "inventory-error"; } @@ -260,8 +269,36 @@ class OpenClawDesktopPanel extends OpenClawLitElement { return; } this.documentSourceResolved = true; - const requestedSource = this.documentSource; + let session: GatewaySessionRow | undefined; + if (this.documentSource === null && this.documentSession !== null) { + // `sessions.describe` is the exact-key lookup; a `sessions.list` search would have to + // page past same-agent keys that share this one's prefix before it could rule it out. + try { + const described = await this.client?.request<{ session?: GatewaySessionRow | null }>( + "sessions.describe", + { key: this.documentSession }, + ); + session = described?.session ?? undefined; + } catch { + session = undefined; + } + if (operationId !== this.operationId) { + return; + } + } + const requestedSource = resolveDesktopDocumentTarget( + { + source: this.documentSource, + session: this.documentSession, + control: this.documentControl, + }, + session, + ); if (requestedSource === null) { + if (this.documentSession !== null) { + this.state = "picker"; + this.noticeText = t("desktop.sourceUnavailable"); + } return; } if (!this.environments.some((environment) => environment.id === requestedSource)) { @@ -543,57 +580,6 @@ class OpenClawDesktopPanel extends OpenClawLitElement { } } - private handleDocumentKeyboardEvent(event: KeyboardEvent): void { - if (!this.controlling || !this.connection?.sendKeyboardEvent) { - return; - } - this.connection.sendKeyboardEvent(event); - event.preventDefault(); - } - - private handleDocumentKeyboardInput(event: InputEvent): void { - const input = event.currentTarget as HTMLTextAreaElement; - if (!this.controlling) { - this.resetDocumentKeyboardInput(input); - return; - } - const previousValue = this.keyboardInputValue; - const nextValue = input.value; - let prefixLength = 0; - const comparableLength = Math.min(previousValue.length, nextValue.length); - while ( - prefixLength < comparableLength && - previousValue.charAt(prefixLength) === nextValue.charAt(prefixLength) - ) { - prefixLength += 1; - } - const removedCount = previousValue.length - prefixLength; - for (let index = 0; index < removedCount; index += 1) { - this.connection?.sendBackspace?.(); - } - this.connection?.sendText?.(nextValue.slice(prefixLength)); - if (nextValue.length < 1 || nextValue.length > MOBILE_KEYBOARD_SENTINEL.length * 2) { - this.resetDocumentKeyboardInput(input); - return; - } - this.keyboardInputValue = nextValue; - } - - private resetDocumentKeyboardInput(input?: HTMLTextAreaElement): void { - this.keyboardInputValue = MOBILE_KEYBOARD_SENTINEL; - const target = - input ?? this.shadowRoot?.querySelector(".desktop-keyboard-input"); - if (target) { - target.value = MOBILE_KEYBOARD_SENTINEL; - } - } - - private focusDocumentKeyboard(): void { - const input = this.shadowRoot?.querySelector(".desktop-keyboard-input"); - input?.focus({ preventScroll: true }); - input?.setSelectionRange(input.value.length, input.value.length); - } - private toggleDocumentScale(): void { this.scaleViewport = !this.scaleViewport; this.connection?.setScaleViewport?.(this.scaleViewport); @@ -622,15 +608,15 @@ class OpenClawDesktopPanel extends OpenClawLitElement { inventoryError: this.state === "inventory-error", reason: this.disconnectedReason, onRetry: () => { + if (this.state === "inventory-error" && this.documentMode) { + this.retryDocumentInventory(); + return; + } if (!this.environmentId) { return; } if (this.state === "inventory-error") { - if (this.documentMode) { - this.retryDocumentInventory(); - } else { - void this.connectRequestedEnvironment(this.environmentId); - } + void this.connectRequestedEnvironment(this.environmentId); return; } void this.connectEnvironment(this.environmentId, this.controlling); @@ -656,7 +642,7 @@ class OpenClawDesktopPanel extends OpenClawLitElement { state: this.state, controlling: this.controlling, scaleViewport: this.scaleViewport, - keyboardInputValue: this.keyboardInputValue, + keyboardInputValue: this.mobileKeyboard.value, notice, picker, credentials, @@ -666,9 +652,9 @@ class OpenClawDesktopPanel extends OpenClawLitElement { void this.connectEnvironment(this.environmentId, !this.controlling); } }, - onKeyboardFocus: () => this.focusDocumentKeyboard(), - onKeyboardEvent: (event) => this.handleDocumentKeyboardEvent(event), - onKeyboardInput: (event) => this.handleDocumentKeyboardInput(event), + onKeyboardFocus: () => this.mobileKeyboard.focus(), + onKeyboardEvent: (event) => this.mobileKeyboard.handleKeyboardEvent(event), + onKeyboardInput: (event) => this.mobileKeyboard.handleInput(event), onScaleToggle: () => this.toggleDocumentScale(), onClose: () => this.onDocumentClose?.(), }); diff --git a/ui/src/components/desktop/desktop-source.ts b/ui/src/components/desktop/desktop-source.ts index a888b57d598a..afb55a0a7290 100644 --- a/ui/src/components/desktop/desktop-source.ts +++ b/ui/src/components/desktop/desktop-source.ts @@ -1,4 +1,7 @@ import type { DesktopSource, EnvironmentSummary } from "@openclaw/gateway-protocol"; +import type { GatewaySessionRow } from "../../api/types.ts"; +import type { DesktopDocumentOptions } from "../../app/desktop-document-mode.ts"; +import { resolveChatPaneDesktopTarget } from "../../pages/chat/chat-pane-placement.ts"; export function desktopSourceForEnvironment( environment: Pick, @@ -11,3 +14,14 @@ export function desktopSourceForEnvironment( } return { kind: "environment", environmentId: environment.id }; } + +/** + * Lives beside the lazily loaded panel rather than in the route module: the chat placement + * owner pulls the chat page's dependency tree, which must stay out of the startup chunk. + */ +export function resolveDesktopDocumentTarget( + options: DesktopDocumentOptions, + session: GatewaySessionRow | undefined, +): string | null { + return options.source ?? (options.session ? resolveChatPaneDesktopTarget(session) : null); +} diff --git a/ui/src/e2e/desktop-document-mode.e2e.test.ts b/ui/src/e2e/desktop-document-mode.e2e.test.ts index b2266822681e..b70e56b7ee02 100644 --- a/ui/src/e2e/desktop-document-mode.e2e.test.ts +++ b/ui/src/e2e/desktop-document-mode.e2e.test.ts @@ -82,6 +82,7 @@ async function startDesktopDocument( expiresAtMs: 60_000, control: false, }, + describedSession?: unknown, ) { await page.setViewportSize({ width: 390, height: 844 }); const gateway = await installMockGateway(page, { @@ -89,6 +90,9 @@ async function startDesktopDocument( featureMethods: ["desktop.observe", "environments.list", "openclaw.setup.detect"], methodResponses: { "desktop.observe": desktopObserve, + ...(describedSession === undefined + ? {} + : { "sessions.describe": { session: describedSession } }), "openclaw.setup.detect": { candidates: [], manualProviders: [], @@ -110,8 +114,9 @@ async function openDesktopDocument( route: string, environments: unknown[], desktopObserve?: unknown, + describedSession?: unknown, ) { - const document = await startDesktopDocument(page, route, desktopObserve); + const document = await startDesktopDocument(page, route, desktopObserve, describedSession); await document.gateway.resolveDeferred("environments.list", { environments }); return document; } @@ -164,6 +169,99 @@ suite.define(() => { }); }); + it("resolves a session to its observable machine and auto-connects", async () => { + await suite.withPage({ serviceWorkers: "block" }, async ({ page }) => { + const sessionKey = "agent:main:mobile-session"; + const { gateway, panel } = await openDesktopDocument( + page, + `?view=desktop&session=${encodeURIComponent(sessionKey)}`, + [ + gatewayEnvironment, + { + id: "node:workstation", + type: "node", + status: "available", + desktop: true, + }, + ], + undefined, + { + key: sessionKey, + kind: "direct", + updatedAt: 1, + execNode: "workstation", + }, + ); + + const request = await gateway.waitForRequest("desktop.observe"); + expect(request.params).toEqual({ + source: { kind: "node", nodeId: "workstation" }, + control: false, + }); + await panel.locator("[data-test-remote-desktop='true']").waitFor(); + await mkdir(artifactDirectory, { recursive: true }); + await page.screenshot({ + path: path.join(artifactDirectory, "session-connected-390x844.png"), + fullPage: false, + }); + }); + }); + + it("lets an explicit source win over the session machine", async () => { + await suite.withPage({ serviceWorkers: "block" }, async ({ page }) => { + const sessionKey = "agent:main:mobile-session"; + const { gateway } = await openDesktopDocument( + page, + `?view=desktop&source=gateway&session=${encodeURIComponent(sessionKey)}`, + [ + gatewayEnvironment, + { + id: "node:workstation", + type: "node", + status: "available", + desktop: true, + }, + ], + undefined, + { + key: sessionKey, + kind: "direct", + updatedAt: 1, + execNode: "workstation", + }, + ); + + const request = await gateway.waitForRequest("desktop.observe"); + expect(request.params).toEqual({ source: { kind: "host" }, control: false }); + expect(await gateway.getRequests("sessions.describe")).toHaveLength(0); + }); + }); + + it("falls back to the picker with a notice for an unknown session", async () => { + await suite.withPage({ serviceWorkers: "block" }, async ({ page }) => { + const { gateway, panel } = await openDesktopDocument( + page, + "?view=desktop&session=agent%3Amain%3Amissing", + [gatewayEnvironment], + undefined, + null, + ); + + await panel + .getByText("The requested desktop source is unavailable. Choose another source.", { + exact: true, + }) + .waitFor(); + await panel.getByText("Desktop sources", { exact: true }).waitFor(); + expect(await gateway.getRequests("desktop.observe")).toHaveLength(0); + await mkdir(artifactDirectory, { recursive: true }); + await page.screenshot({ + path: path.join(artifactDirectory, "unknown-session-picker-390x844.png"), + fullPage: false, + }); + }); + }); + it("renders inventory failure recovery and retries the preselected source", async () => { await suite.withPage({ serviceWorkers: "block" }, async ({ page }) => { const { gateway, panel } = await startDesktopDocument(page, "?view=desktop&source=gateway"); @@ -187,6 +285,45 @@ suite.define(() => { }); }); + it("recovers a session-preselected desktop after an inventory failure", async () => { + await suite.withPage({ serviceWorkers: "block" }, async ({ page }) => { + const sessionKey = "agent:main:mobile-session"; + const nodeEnvironment = { + id: "node:workstation", + type: "node", + status: "available", + desktop: true, + }; + const { gateway, panel } = await startDesktopDocument( + page, + `?view=desktop&session=${encodeURIComponent(sessionKey)}`, + undefined, + { key: sessionKey, kind: "direct", updatedAt: 1, execNode: "workstation" }, + ); + await gateway.rejectDeferred("environments.list", { + code: "UNAVAILABLE", + message: "desktop inventory is temporarily unavailable", + }); + + // A session key only names a machine once the inventory loads, so recovery here has no + // preselected environment to reconnect to and must retry the inventory instead. + const retry = panel.getByRole("button", { name: "Retry", exact: true }); + await retry.waitFor(); + expect(await gateway.getRequests("desktop.observe")).toHaveLength(0); + + await gateway.setMethodResponse("environments.list", { + environments: [gatewayEnvironment, nodeEnvironment], + }); + await retry.click(); + const observeRequest = await gateway.waitForRequest("desktop.observe"); + expect(observeRequest.params).toEqual({ + source: { kind: "node", nodeId: "workstation" }, + control: false, + }); + await panel.locator("[data-test-remote-desktop='true']").waitFor(); + }); + }); + it("auto-connects view-only and provides four working touch actions", async () => { await suite.withPage({ serviceWorkers: "block" }, async ({ page }) => { const { gateway, panel } = await openDesktopDocument(