diff --git a/apps/android/app/src/main/java/ai/openclaw/app/gateway/GatewayProtocol.kt b/apps/android/app/src/main/java/ai/openclaw/app/gateway/GatewayProtocol.kt index a0a6a596b559..1b0faff3a0cf 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/gateway/GatewayProtocol.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/gateway/GatewayProtocol.kt @@ -139,6 +139,21 @@ data class SessionObserverDigest( val planProgress: SessionObserverPlanProgress? = null, ) +@Serializable +data class WorkerDesktopObserveParams( + val environmentId: String, + val control: Boolean? = null, +) + +@Serializable +data class WorkerDesktopObserveResult( + val transport: String = "rfb", + val wsPath: String, + val expiresAtMs: Long, + val control: Boolean, + val vncPassword: String? = null, +) + @Serializable data class GatewayEventFrameStateVersion( val presence: Long, @@ -511,6 +526,7 @@ enum class GatewayMethod( SessionsPatchMany("sessions.patchMany"), UpdateHold("update.hold"), SessionsCatalogStartTerminal("sessions.catalog.startTerminal"), + WorkerDesktopObserve("worker.desktop.observe"), } enum class GatewayEvent( diff --git a/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift b/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift index ead4e72a8200..4d9e9058e99b 100644 --- a/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift +++ b/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift @@ -1803,6 +1803,7 @@ public struct WorkerEnvironmentMetadata: Codable, Sendable { public let attachedsessionids: [String] public let tunnelstatus: WorkerTunnelStatus public let error: String? + public let desktop: Bool? public init( providerid: String, @@ -1812,7 +1813,8 @@ public struct WorkerEnvironmentMetadata: Codable, Sendable { idlems: Int? = nil, attachedsessionids: [String], tunnelstatus: WorkerTunnelStatus, - error: String? = nil) + error: String? = nil, + desktop: Bool? = nil) { self.providerid = providerid self.leaseid = leaseid @@ -1822,6 +1824,7 @@ public struct WorkerEnvironmentMetadata: Codable, Sendable { self.attachedsessionids = attachedsessionids self.tunnelstatus = tunnelstatus self.error = error + self.desktop = desktop } private enum CodingKeys: String, CodingKey { @@ -1833,6 +1836,7 @@ public struct WorkerEnvironmentMetadata: Codable, Sendable { case attachedsessionids = "attachedSessionIds" case tunnelstatus = "tunnelStatus" case error + case desktop } } @@ -2042,6 +2046,54 @@ public struct EnvironmentsStatusResult: Codable, Sendable { } } +public struct WorkerDesktopObserveParams: Codable, Sendable { + public let environmentid: String + public let control: Bool? + + public init( + environmentid: String, + control: Bool? = nil) + { + self.environmentid = environmentid + self.control = control + } + + private enum CodingKeys: String, CodingKey { + case environmentid = "environmentId" + case control + } +} + +public struct WorkerDesktopObserveResult: Codable, Sendable { + public let transport: String + public let wspath: String + public let expiresatms: Int + public let control: Bool + public let vncpassword: String? + + public init( + transport: String, + wspath: String, + expiresatms: Int, + control: Bool, + vncpassword: String? = nil) + { + self.transport = transport + self.wspath = wspath + self.expiresatms = expiresatms + self.control = control + self.vncpassword = vncpassword + } + + private enum CodingKeys: String, CodingKey { + case transport + case wspath = "wsPath" + case expiresatms = "expiresAtMs" + case control + case vncpassword = "vncPassword" + } +} + public struct SystemInfoParams: Codable, Sendable {} public struct SystemInfoResult: Codable, Sendable { diff --git a/docs/.generated/config-baseline.counts.json b/docs/.generated/config-baseline.counts.json index 750b749e8e8c..3507a5f26813 100644 --- a/docs/.generated/config-baseline.counts.json +++ b/docs/.generated/config-baseline.counts.json @@ -1,5 +1,5 @@ { - "core": 2326, + "core": 2284, "channel": 3694, - "plugin": 4056 + "plugin": 4053 } diff --git a/docs/.generated/config-baseline.sha256 b/docs/.generated/config-baseline.sha256 index 17c2f1e7a237..cc1ce46fe867 100644 --- a/docs/.generated/config-baseline.sha256 +++ b/docs/.generated/config-baseline.sha256 @@ -1,4 +1,4 @@ -467b81275246a1512907edb55c00cdb1bfa7b929977cde5c190b97299639249b config-baseline.json -fdcb5717167f7ac0200281a5d0cb5362fbaf8c6b5f51f866958f85683189a46f config-baseline.core.json +0ce08dfbe1fb23bd5bb03ae12f46fccff569b0aaf374b3984f1fbcb61d2f6b66 config-baseline.json +8cbea1bb4c3c0bd933da5641c08a3086e4579d8bcc496468cc951d7e388978c9 config-baseline.core.json d752cf8a7ecd2d31684557aeae4d2c08f2ec4ce5f091bce32d54a4685eefdf45 config-baseline.channel.json -6df755fceeafe28ded568b6f4bb1bd0bd49050a601732e52aacf4ab56e398b19 config-baseline.plugin.json +7ce54344ec431d11f8a1ac92a112a9352fc55a4e43be71a08b91f9a06f924fb6 config-baseline.plugin.json diff --git a/docs/.generated/plugin-sdk-api-baseline.sha256 b/docs/.generated/plugin-sdk-api-baseline.sha256 index 6543cb8ec418..5edb90ca1d0d 100644 --- a/docs/.generated/plugin-sdk-api-baseline.sha256 +++ b/docs/.generated/plugin-sdk-api-baseline.sha256 @@ -1,151 +1,151 @@ -7d306f95a7f8c3ea36ba68dda7d121c4f93fc9f6326c6c53683ab01eccf0a2fc module/account-core -c38b59ef4745b7447295baf17900078bf460fe36d98b3516e0f37aa2c25fda5c module/account-helpers +a06111c1cdb17e9ea1f2bbf2f90b86b8e93150c091b964b6fc5f5e67f5a4b632 module/account-core +d20eb6a6a77f72566266b88d191b30155757a1cc3d3886d3b6c437176bbac771 module/account-helpers 71522995185b956a0cc4927a472cc8d1153e5e998874bfd9a750513175174713 module/account-id -62e64563d598ebb16411a0248da2f2d7fc61ca42602335c54b27586d1a4dc3bd module/account-resolution +006e581db74e461dcf4d7fd299b09d285f32c91ac12a56de1d21307d272e5a0a module/account-resolution 3fe118210b885af40088457ed81ffa5ede18c8e695295731a2ee059af46843cc module/agent-config-primitives -655a3ca43aa8f2a9e14599795abf743b68fcee9b9a59187c636fee71eadef12f module/agent-harness -b6813520c32ddd533c86874430d21ed78e095233ae684573a59c43e8da6b3d87 module/agent-harness-runtime -773943f0f5cc26d4bfd1dc6cfdac5effc2bec14f1bde927e9e407ab4c9701ba0 module/agent-media-payload -280e0ee3d5548913c6976d57ae88953cb4ff586efe2e430ac30ba89ca5296dba module/agent-runtime -241d467d0af5f81d8a535fe0c65d226356daf8325a037cb23092b8dd82aaa456 module/agent-scope-runtime +44c9f3652dd8d8a8aff03346ee991b81ff8d4f3f3de2c641935edde4bf7501da module/agent-harness +826caa6e66f3df66bb5b13bca269047145a44145271bcf3e54c41a2d54fe7b8b module/agent-harness-runtime +595ef30046b8ba4bfd427e5a33b65ca28ab1a699ce7e90d6afa2ba2b187e22c0 module/agent-media-payload +453437a1c945a732a3faeee79f3f3e6b505a37d5b353878a91512dd140565ad3 module/agent-runtime +72fa7e17dd1694a24f373dece9d3035c3fc25e1e113954328e16df03d9be7b55 module/agent-scope-runtime 8fecb210e22bce4532b6ab649b09465f0bd2c857a44abf40db7d683d6491e6da module/allow-from -ad09805cfb46d6155fb54807ea9794ec3980a5eac62a9d1e4d718dae8d0b61df module/allowlist-config-edit -05e9015f0b462f67be33831ca58e31af3cf99a13ec103a00c3111bc1eff989b0 module/approval-auth-runtime -f06655c8a4524497abe4fac09808a3c18c10260fdcf90f97698efce6cf5d62cf module/approval-client-runtime -5cdacacc7cb9950bd8fa3b4332691adf36e69cd3f9e2928dc59e70897560f86e module/approval-delivery-runtime -0efcfa871dfeaa2f4563ca5d178fc4df4c1a397c635e1d0d802735073875afb2 module/approval-gateway-runtime -5da9a30393531c72d5df3ea256eaec71ca7ad323ffe9539d27a7bfc4c75d279f module/approval-handler-adapter-runtime -f09c3f27c32752980a135018353c07260153c3df7601da39fe7326eeafe515e9 module/approval-handler-runtime -040dd9bbd3d1235c4679fcf219bdca65551f014b6ef08eb94935399f3d83972f module/approval-native-runtime -573a09bfa745b9128aeb73cf736ff0e5718c51ea5fad050bae795ed31c8120f3 module/approval-reply-runtime -8209d1ee046f37bbbdeefe2733a560f3e2762dc0e2b188399fc8147f35a9391a module/approval-runtime +ac6c71c3b7c3d1f6d3219d7514a1d1e0c44198358e6e9a4eae06e318d52a8433 module/allowlist-config-edit +136ddb70973082ec051eee5a5c2a78da5359f6d0e2dc058be5f7049ad5d2608e module/approval-auth-runtime +cdf0f3cab2f9e19bf9a7b9125872b4e9986bcd0530d17a3bdc76ec00aa9d8388 module/approval-client-runtime +235abceca3e9392ab39e5e63231337ae9dc35bbb773fb79e0e5a0b56be403455 module/approval-delivery-runtime +b49c730d28ede4fe15732a30a5cb6f38fd0ac3246bcec4a0784f0b2f2999ba5a module/approval-gateway-runtime +9750dd4c52ac4b14142be0d39205da33faf584fdb48bb4d2942972e608147ccd module/approval-handler-adapter-runtime +f1d44213e7e5d1c8d23499b4f634395a6827ddb390296a9c63f53acb1c108b02 module/approval-handler-runtime +a0904844a521a3391cbb183e8657b69f7550b6245f88a7bcd7b5c12b2e3fae24 module/approval-native-runtime +343b855a38a460a8c3b65b005698ed5c311e9f6b93e7d42a142c5cddb3a08a6f module/approval-reply-runtime +fabf4fb018bd3c8a38e6490e28bacae0e1fe9795ea3c334c845c6e23ad95142e module/approval-runtime 01ca912836b8dec672f705e294f72d346e778557e4c591317d67558ea7669c0b module/archive d7e53de63b0ac11a266e4abdc18ba6e9401b80309f5c8f5f6a72a00f65dfe3bd module/boolean-param -39769190fc9d790ae5e2c4ebd5de0a78b45b3d529aaf264c84eba4e7f86a3066 module/channel-actions -54160b43e8382b33667384c56492c7d1a9641795b990d026821a4c1a588d0126 module/channel-config-helpers +1b87d321f91ac51be8010cfd45163e1b1f13e4f68de60539fa8518f54258b4b5 module/channel-actions +665c32fdc411e106469713fce1862d4b38287723dae1527b29bb50703f93a3a0 module/channel-config-helpers c2cc71d5070b6071c51248b0648d1ad1a9468d3737df890adc77ec02025e8853 module/channel-config-primitives 76ad615d374431580ea1755594e2ce3ce047ac1de9fd4621053dca0fbc3afc4d module/channel-config-schema -cd873744a01a47c5284227d8af5c50d2f6606a815e8586e11c2012f9c301429a module/channel-contract -b6a7748747f6c2625c770549d04b2d7a9c4652fd80f3cb7a075dfd6ee93b7d6b module/channel-core -caa54fde5a2a515491019ec163ed278e09bdc95b9de9926aff04c1743ca0c966 module/channel-dm-policy -d4122a253000d5e95ade0d97a29dc1a8108c124b0b21dd4940a0c8f02d48e079 module/channel-entry-contract -2c55b3f3d1d275f760a7e1c78764e4030e7a06c4273a4d51d8f3c82b55ea818d module/channel-feedback -5524c9407d2f01a8262f18ed34084d83f7c96d5de16777daca53858485213cd1 module/channel-inbound -d7e21bb831ad5125e6498e88fcc27e820ed296784eb2296c22795c08d55ade06 module/channel-inbound-debounce -79a8f244b0627ce4b601231bf71f0ee539f6ac7692e490847ec8e7ace7d29f1c module/channel-ingress-runtime -e8e08fd9dfcc15758c552fd594055cde15457fac07688553e613de27a6909d98 module/channel-lifecycle +63d4f5bc22d6e7779fcfa1f73e70df689189fcc876af26e05a4e3334b95675a9 module/channel-contract +0c82d1cd5c0dd76bcd98dbcc103032cc9d93eab817c8eff6cb02f8a16bc41046 module/channel-core +f6ddf9086bc224b4b1a516cd7e056467a529d19435930b1c8ddff7052d890ffa module/channel-dm-policy +32aaf9a8ba28eb8281691d8dd5b827a87ad2e75ef57dba14b004e4a5844d7856 module/channel-entry-contract +b14ed3e3235fab5725a5eac86bb7910174a21ae1d7f2b4433891867396dcc566 module/channel-feedback +a4f2d63e4b07d04c26676c632137853f927b84993517ae77caa9ca2f5e8637d9 module/channel-inbound +f58349b93db16be763c7ba4c00cbc13b1b64911c8eefa5a1bfba1e95809e81cf module/channel-inbound-debounce +cb7f865c9953b5f0925c918b2bed29fb6629fbd643f4f69bd2bf345970864cbf module/channel-ingress-runtime +af775123f5526907732b7bebd6e2258ffe4ec733b2435007361bf7aba5b4eaf4 module/channel-lifecycle 0e47457e38d1df0bd572e1408cde2ca6a788b65205f43c585316b5ad3a8f2f16 module/channel-logging -b1726a02b686a1c2290631264e577b3bac9a389cb40610c4f781680bb4e69eaf module/channel-message -a25750a5166f24f6ac34f2bf9f3b2fd2d607352e4043f0414a642627036b12a8 module/channel-outbound -f09c4649dba58241ef0f58c2b3e505d726fc734a103b7b350b9bd35e4690a9d2 module/channel-pairing -092c76e9a3a1e211e231679829c993630fa67064fa7f0ae8fc5d6e8b8630034d module/channel-plugin-common -39e460870b572913c321680b6a4ca2fed963b8c8c71bc2de9193ec68b74ef12b module/channel-policy -dd291daf278dbe9110ad9007048830b6f0bd6701dca09e25094ce98e4642d972 module/channel-reply-pipeline +65a1339bc4d3ece7c41f1e7d910af87bf29b6bf372bea3647f76eb3c4f655ce6 module/channel-message +202248c5cbb996d31fb7d7584d2bd6e91dfa9f9c0da7391204636f6e5fa870d5 module/channel-outbound +fff8d145ebfdb345560f520dc8f6a0d180f51327a50abf137548e6a6c3651a09 module/channel-pairing +04911fcac9b4fa626b8dff768ca15ae3b7a999ce913379cf6078c010ab578ef7 module/channel-plugin-common +113d135501f2777f308c4c3a59dc29f722f8c385ddfe5628994f14ed37246442 module/channel-policy +c3bdfac92ace16eccca1bd44c95a3e73e39d7f996f1714cdfdf102647305df55 module/channel-reply-pipeline 482370e60135db9bfaf07f24bab549e5fde09ab265a6061a1f587c5d93929e91 module/channel-runtime-context -04948928b3cf310c4c401bdb89e94ac2ff2066b5b464a9bb501b36dbe7711c07 module/channel-secret-basic-runtime -e99d6f57a89503e67035da553b0ae62b893722e564031c050f3097cf5fa3bf5f module/channel-secret-runtime -57d12e9b62cfb09142bf95820b670b7ee9f1172fb069a9afdb7a62f7c30c598d module/channel-send-result -8337cefa180c854ca943642f9e1c50886dca0ed879d67be268ba84d0bc4e0cc2 module/channel-setup -24cc60a72cf009f02383b5ee1a787dacbf6a8d87b192375f1a51b5f04036825c module/channel-status +121aaced4cfdbd2dd16462e685699d021107d5d1bc4f9324363173c086b72605 module/channel-secret-basic-runtime +965cc7241ab4f6398611052f873f06b47b996fd72c948ea973c2cac3eaf4a820 module/channel-secret-runtime +9d25865ba44e3160b8716b08ed0ad93ab95e2c1bdde28047f65dd03fe66efd83 module/channel-send-result +b89224a72be296c82eeb1933b179ec0479627e80e1c61717b6cc7a778dbfa5b2 module/channel-setup +329fee4cb0ca430ea9249e4090e889c459402a40d42acd9c0f2a20053167502b module/channel-status 95dc206f832a0f563238dcea72d41554f409a3a9df081f41c52a8241c7dc5161 module/channel-streaming 67df67da5ae72e9eaeb19d41b6bd2432ec4fd8b7b63b2b616fb98f3b4e0ec41d module/channel-streaming-config 1303df5cc58539c6941e2cd159c93259804c925795219f1630f4d740896a77c1 module/cli-argv ad12670dbfe538f8d0ebf4fb2b68080e93a760278278e6b1ce9bb129d4b2d533 module/collection-runtime -0e0af5cba658a4f4c883cfb80a951c8dd7e56ec5fa5e240d91703801b98b9718 module/command-auth -f3326daef3b94b58c39bdac5b0596690a09cfb59a4c703d15a077a726e5e9097 module/command-auth-native -a41d9effc1656cbc131611099bdeb81423ab48adce8c1b1ba07100dff8d32818 module/command-detection -31044216c6495728a36dbe60da31f7c1ede2b069cd0d0685d887ebca31814d5f module/command-primitives-runtime -e42cc234ace96a64cec0bdb8051ddded77618e384ae2bbce8b633bf35dbd0288 module/command-status -91f95003a7ce8d78af219f6997e3001a7690674c2682ffe987942d3b1f45d8ea module/config-contracts -5e866c4f8dea30045a8a94e037f0e202d597afb1ca221d9854b15bc416503643 module/config-mutation -a5328945c964794236201aa2c947d783f210f49af34d213d5a188b1e6c40b1a4 module/config-runtime -c6d742c9e6027647502399560b4264bed80bb8c069c2d08ba8527336f58dc13b module/conversation-runtime -d12aff6806e9492db176b818d8dfe692d4e53f5d310ca05b23ada9ed45c9833c module/core -3e1f68c5f5e063cab69565471a3c8908fc2f099d38a992effe36dac79fe17cf5 module/dedupe-runtime +d7adab8dd324cbb7f2946d6819d50f9ea1c94521cb45ff8fc1bde722e61010db module/command-auth +69df21c3d335cab4b702a15b63fdb7861407380fe304501803f803bb4dac465f module/command-auth-native +4db2a3623b116976a2e4cb366163194a8073c8d56a9c74105127bee59c7d07f4 module/command-detection +0f6cf0b06cd65f2bfded9d1054873166fa6ad22c71f4f1cc5afb7a49711cc365 module/command-primitives-runtime +03eb97e7b47a79ad4b40b37a2bac418c14f85e90969f70fd6fc5c80bb27b8c49 module/command-status +b56349e8d8034152be55161d4bb4c9bf69d0f68f7ea07431f28a4ee7a707d576 module/config-contracts +9d2c42377ef981ea6732f96bd3a9c1a16fa2852936516a3fca5adc2b6b0b736e module/config-mutation +6c70300fd4bf84808f742687eaad92339fc942abd5cfa2378807a0756decdbd1 module/config-runtime +d5a157da395c2b548347b7fae054af0a90afbcae71c7171d0be5c66d92271328 module/conversation-runtime +4f07c051c9bdca05b20b726388d808baad77f5e6078de5e20a66a2ca160b2b3b module/core +e2c79d89d54d9b94fe8c633995ee1610d1db88e6f14f52e7df49226cd245148d module/dedupe-runtime ebef0e650ab45e44c9335e2b3e15588c968cea6dadd125364a076f9c50ad1e8c module/device-bootstrap -fd7c489415aa272af724ac15e95d297eac0edeb229e30d02c4061d5276f874b2 module/diagnostic-runtime -6dba2e37cfd962cc8c0aaae2d74d82407fa660fa56a4cf146b8674dc64239608 module/directory-runtime -b3a0c1a204b053f66b2b20d35dbf6b0f9547cd8fb4403a164d8d88fd74d49d05 module/discord -c468c0ca5e5fc093ef5bd0cd15c57e19059bbe5c453d68f2664e9aaa35661cab module/error-runtime -6d9b6396888d7cddded108e211053559b10d79397d20098e0616767be4a4bfb3 module/extension-shared +4fe9beab67598950c144d86add5fd3b1ce475aaf37644d2f9798703052d98210 module/diagnostic-runtime +734898717c8669f1c3a35dca93db121508368edac7dafcd1ccce71f432680979 module/directory-runtime +7b7c8cdd58543974f47eaa8c906357b6dc7c0fcbdb26668300715e190349ed7c module/discord +2b01f2e5a52713158665372b358fb6059903f68b6efbedff700741d9feca7696 module/error-runtime +03eeafd10471b94a2651e0d42380a32f5ffd4f33c47e4b0161205093163e9407 module/extension-shared dd9f6e0fd33cc88b22543c1ee30cc09cf4de4d8f30dff7b7f9cebef885c21543 module/gateway-method-runtime -c8b2e9e52ecb5be22686eddbc282c74a0eea15e399b41dd4e7be817ccbb6cb0a module/gateway-runtime +64c88d5090e478fe9a16fb6845f8bef2b804d4590f812b6f2cf1ec2c3c857306 module/gateway-runtime 575656e5e0195c8d1813a4e2e3a271e800bb97d44f2dd78c242e3b6714ffa097 module/group-access -92566a68cbf1c635fe3dc29afcab042e2b06aaaa0438c8cbd2c001b07730aa9c module/health -70abcc263a1f320faf7e589ba238bb1b4c7a602d52af42461420523460804aec module/hook-runtime -185a5acedbd7f1a73e5cc773e22bf494b124a09bcd68b5a756ff0f881abf431b module/inbound-envelope +f3595668fc4c20b2b34df67d3fa4d5432c16fa3be5fec9e63d8f14512ab6732f module/health +60d126e420e212415f25b6e90c2aca7513a1275e14b2bad27b06aeb6ab12eaa5 module/hook-runtime +da9d83537008db2c9339f3a4235ed2b7ee3ac986cacdc8bf8fdfaf1a025bd6a9 module/inbound-envelope 4928af5d2509f696b896f53ac790303a0742202dbcdae3e44fe6d1b434a9c1ba module/inbound-event-delivery -b8fc15e3e1094e8f16f65f48ab6da11e2ad8f835a414668df572f3a910abc1d1 module/inbound-reply-dispatch -ee37c72512c245c1eb9637e8e9a788f128749fcb5d0308d45fa442a45dfc18c5 module/infra-runtime +4e44ff8d7fa64fe302f081cf60c29a12e579ea55e5e6b6994bd3c918910a4fd0 module/inbound-reply-dispatch +e5fdb21e7d557fb6830f9b7e9860bdd9b11f9ceb8c34e59b85e12a5ec5b022a3 module/infra-runtime ce73721421f1b903dd04ead4df173582e59ea3e9990248102c448b419cc6d272 module/ingress-effect-once -dd7a5a732737c74cfe287ab40d62dd380ab4a3b78eeea6cd52574c0613a874cf module/interactive-runtime +31449afd7ea7f0c8dc7dee439a3693c3d1092af67c6bed67dd34e7b7b50bc74e module/interactive-runtime 408d257ab5cc4b88a22b7e7595039cb8fc524b261c44141b294fbd0100ba62ee module/json-store e907fd3a98185f2c261f2aafcaa5a19ee1d7b459d519a498397d629f84c68312 module/lazy-runtime -3a6d4cd20932d21e5ae665320ff594046c656eb84d95008a318ee1e9793edf2b module/logging-core +09735d3ed2b372352a4e32aa8e80d7b8ccf7f78768d2fdb76d00bbe9aa659f7f module/logging-core f1ca4ced4305d0769c2d8cc1291137ac7002fe0e6eaec2c1a71edad2204c8311 module/matrix -7869c8dcea3b96ab00a33fcbd21a6ca171131b0c7dc6a527bf1da178c092621e module/media-local-roots +c0a6a1e65d02a5cc8727881a69d8f4ec692a33dc78fe13320d8372c33b4b75ed module/media-local-roots f74d7295fe716aa140aa0bc9300d6259d71dab826de0808fca6bb02592bf5d6e module/media-mime -dfddf0032904cf003c325578bc4b2789b2b695c4f7aaa9d08d2f932df30477cb module/media-runtime +819c3b13113fe8cce4d9792f1b649d3ee48e9a9ab2f44936f553c294fc4b8405 module/media-runtime 6a52f93107335f88751704352cc01e62add06f854a5b7d765e2a5ee87c0313b6 module/media-store -7dd4a69b33196e946b03a3579e3b569176ff32db22fa668d5eacc06227574cd1 module/media-understanding -151c5fb10718764a03a913b3d6a8a6716cdc12fb95010f306b228a512d3a3d91 module/media-understanding-runtime -46cc7c1db3c4fd02dcd75c2e78b2ff9b3f50a2dcdac4a04104d03e739c18d43f module/meeting-runtime -d16cbced4f2e6672ac9a032ac41691fe7ff4994e328d44ed6ac8a46dbbec834f module/memory-core-host-engine-foundation -dcdbb62f474fc7167b58a08f9276e43c380cf6680b95f36f4624ac19e79c7a76 module/memory-host-core +b3fdb9f96d2724d5824063f434f3e482b605e4c06cad9770a94c78079dc36454 module/media-understanding +544a6d47a391e64574b146f649e8065b70a57955088b85cbdbdacb872889475a module/media-understanding-runtime +1157ce402e4c67aa712c302613d8db383efb86fae4451cf2b349a0d00204c746 module/meeting-runtime +a6aac1a3f85d3ee7dc9fbc7a1c6ef7dd00a9c2d3bdb6dc8d57a9704a821f806d module/memory-core-host-engine-foundation +2f9735420972828fc0a577c2bbc73594e6517ac0e249c7ada3fa8d2855df5db8 module/memory-host-core 1efa0aadc4261d1c6073058cbf3dcc9fa681424819bdd14333e19b249bbc4b18 module/messaging-targets -a3c8b86036354d27ae5c669502de05b823f3fb5fe4ef06a4defd4ed1fbcbf9a0 module/model-session-runtime -85bc84c819c8ae70ade9acfea6a9adf4b728c1111f1c6414c0ebeae6404a2dc6 module/models-provider-runtime -06b3bd19f3dde06b77cb1b0e8e389a0bc69b90f2aca4da86ddbc0e29ef8a33d8 module/native-command-config-runtime -faaa22538f3459cef412c52088cb40dc732aa560fba2e70655938460022287d0 module/native-command-registry -5b968ecbef95fda927d0944409994e355ec878608dd084358970078d2ac545a9 module/param-readers +13ff5bcf2fd30080a72ff51f32d9f2bf87262f84547430eadd923b8c0c9a2152 module/model-session-runtime +c766f8ef469c702f222c4b9cba1ce7417659f0d643272975c9781e93e53b8d57 module/models-provider-runtime +504c61546d566814cda2d8126b458efc9ad70b75f1406ab9d6635ac10225200c module/native-command-config-runtime +d808e6681668e70b2dbb2ac590b455072f98d54970593e25ef6ad5c2322d3a51 module/native-command-registry +ca6ee4fa75f976d590210b9ff6dc66374bb829a05d4f70972eef9137f5548b88 module/param-readers ca7a56bb1a6169b4cf9befbf5aa21da280a8086fdc49fca4eec520a7a7c98549 module/persistent-dedupe -1bf4d4dfe5a4b264cf6fb8fbd0c7bc76f520ff9845cffad6da4b3a3c2bc3f6f6 module/plugin-config-runtime -7e76313652969a471c2aad275cbd12a9ccba4dd4a684575b631df5e4e9394af4 module/plugin-entry -bf9c377138073b793ec8f0ac6057be6118491d07d83e29f916745a2df1f21285 module/plugin-runtime -ab64a67713848846d92b4185506682b7065ad358a7bbd338757afcff0e6da593 module/provider-auth -5de81322e7fdc40979805d070de620a41c75476223d07003d4fa74b61f1b667d module/provider-catalog-runtime +3c73ab232d86b49ebb6e5f302da2a2a4b5c7f62bff5e4cc0d3505edc5ba4f5cd module/plugin-config-runtime +97c652cef9e02251824ef20040a87e20df403ffe11ac18c39843691b96479c83 module/plugin-entry +747f801f7f819f6c88894bbe170b3e10a3ad5d74648d210f13947b37dd9fbc74 module/plugin-runtime +a6689f724866c7b107c7389e36dbecb640f1b5ea8dbe802191e009fadcfdf7a0 module/provider-auth +b5eadf1710edceeede0fdd7f64b8b6973d507cd71e6a73a6f65c8055eff0cb9a module/provider-catalog-runtime 8131147d699394bd06503e2ea2f5f1a50b1594a87dded6d118b74a8d0328c8f6 module/proxy-capture -784c3c5c5dbb1e2c33ccccc62f850b740d2adcbde5e10e4e891d8c0f78aaeb99 module/question-gateway-runtime -158d7fa58b45efc8569684cbb8cd2d0a8e9d331911c9eacd6e1323c4b764cbb5 module/reply-chunking -37603be66985dad7b4ff39d3c417bd497e8a7ed65f980697ff8c9b28439296ca module/reply-dispatch-runtime +4949fe3958d3b92b8d2e13ec0af9e65c88e1c7d3e39e70c306fd78c93b094e87 module/question-gateway-runtime +a479cd5c96a34c6f0a2ed4d4239aad63b56cb0e47a970848a141ad7fb67b9e11 module/reply-chunking +2656136a48940e5f78c1ff71c63e1d0f776b9cd36320afab8a4b34c58767acaf module/reply-dispatch-runtime 73f861fa3179d5af1159853c5acab0eec7a6c8f9398dcb75ea770e784fca6727 module/reply-history -c4633871d5982f7b3d447ea77afa90031fd2faf90750f6f5dea8367e3f57d52a module/reply-payload -a79af52d2e350a1879d7ba2453b713edd66b4ca4f6e9e48377e9e9a7624637a3 module/reply-runtime +d3bf7e4a7fcaebc1cf3173f4fbb60203e6ec46257cddcca7096800df946708fc module/reply-payload +b4b01506b56fe0b174533f71ed50811bab26f443112d5e20cf5c4c7889c02b4a module/reply-runtime aa07d85d99fdd2b1e0cbe9975fb6dcae66b8bdce2607c6bd5402ae68bb15118c module/root-walk -e26cc92679c768fa1474f15828f545aec87f32718a6b35f7907c4f56f65542fc module/routing +02c0e5cae6772159a1c5ff43209de542ac7976dc30541d406f9dc6c37141a69d module/routing 7877a7e58fa32a64107154e5b714c6d165e96989d4aa5f43e0afac085a187af0 module/run-command -c83779c80c9e7b196b31a39ec4b986098bd701b089bf6eb2f53c368fb982ea77 module/runtime -159b563aad773cef67f18bf9bd2653420bec94b599e052f2a1d5a238bf341e16 module/runtime-config-snapshot -9fe5bcb52b462010214eda1c01f60b3a018837d9f95dc864f6d457cb3da001cf module/runtime-env +bd15eb9689fd7070dc942cb4bd0db6c2c1d4fb1a971ce9d0a6349a5679309535 module/runtime +ccb6aad96b4ea156738f45bac04b6eda5826e17b36eb8b5e26567f400b3be217 module/runtime-config-snapshot +2e3c692a9f911ab227e34a9a3870f8e139f90ceaff631301b55ce712dec53736 module/runtime-env 7e871b7319745678bb83fcfc1b54c8751b0ab1af92ff06c01d0659ac92863c11 module/runtime-group-policy -8c8dd7ce1c979fb668bd8b526995badca9ad6433f1a5ca3f8e2af4df1518c98f module/runtime-store +8e0b2bd21503347e252c5a53f21e2ccaaec1abe03eb9374c2dd3dccddd7098c8 module/runtime-store d17862c40825af1ddf0257b44f1e1cbb9c375e8e5ed668fae75d530d1a465cf9 module/secret-file 8e2ac4d3973d8d8ce4478e3440d66ee5c0d9213b0fe9e927c421d14fd31e5e86 module/secret-input -e7672788f052a1249839c8fa24d6a9a45967c4e3fd94e890b0f90296026cd1ca module/secret-input-runtime -026631cbf010d0325be2c4fa4ccd8bbdf8e1008f0904c742c270811f578b07b8 module/secret-ref-runtime -536d3196e17422652d755286b9921133562907537c906b067a568b874221e7a0 module/security-runtime -7d6ec8117cab188a88a22f0555e50620a4c18f4b78aefe667db670268f69d528 module/session-catalog -89886db92ecb3f75e0565586b55bba1f965eac46be55f46c23838584c3140958 module/session-discussion -7262e6e6dee725b4d9b8226bbb31f24460d7a4d5956aa6d60df81181c4221c07 module/session-store-runtime -e68e6edb57b7dc978431495ab53712d38a02d61620ab272b46a38c3b94199cfa module/setup -f188bdb868523aa17457c9338b02fcdf0df548d2fe776750b39d36c702edc061 module/setup-runtime +1806dba733bb6d88b4e997d3dc989761f0d7dde7638d96e1218705fd27f0921b module/secret-input-runtime +8e0e6d67db89eeee760a33ac984e7141bde171080365dddf534858b6abb3d56d module/secret-ref-runtime +c810981c42d32923e84c42d20137c9dc393aabc0ea079786e01d19bd59fa5277 module/security-runtime +9f353b79030b77b8abadda98a7b3acf05eb1545b2069571fa632e0f658b8e225 module/session-catalog +485ecc02b7aa91d61b292f3a374b62c328f63cdf326f607774cf0f2d5c5400f6 module/session-discussion +fd2dcb08a59df7bb68c8f36f272d134f6904d12b9e192bc8d7a4858d3039b595 module/session-store-runtime +73f4a776c027d974f010250360822694486a8033bf1b51e96a0ff2440b05dbfe module/setup +f1ec91331ad72c3fd9edb2cb11b4c6a35acd55ef3e4f541c5907149eed21b15f module/setup-runtime 44d37e0d9131ad2859f41068f2604090c784e65f1bd6ebda8e051b6f2e5e1660 module/setup-tools -3601dd9f28de70915005457146881685b73690ef82c9bfed80f35d0cf9b645b1 module/skill-commands-runtime -297918bf72f8b65b3d6be33268c3cf3eba7370531b4eae031aaa078a9ae6bc1d module/speech-settings -4a1bcc606805b5a20d04e048fc268764df9b140be9d161ad981d213343b87fd6 module/ssrf-policy -3ad3f12186d9cf44600904f22d0659b3a72737d0ce7fb33d1177372f44db827c module/ssrf-runtime -5501c65f90feec38049ce100cb320b2a629ebf1ad04b21f015a0d44aa1e4c448 module/state-paths -6422d1324ea329a670e357e522dfa28f0b3c6c2fb006c71d7fd75f8dd6bb13d7 module/status-helpers +00e8794c6e7aabbdeec14189d885d4021f5118fcb08f7e34b85fafc473e62bed module/skill-commands-runtime +f7617584de44dd87e55b0871f27db49434c29f4cde76ecfb8daa3d817a965346 module/speech-settings +f01b661de86de0d0b1d3bff395330092f3fe1114f4516968ca5fca79cf7eac50 module/ssrf-policy +0296f2c837f8116aa3a7e8b02d3de8b756d44e0a004842dbd24a5aa627f74a54 module/ssrf-runtime +eace34246d7a827b00a67bde258f8401bab35f527dcad29e6f982e4ae39cf013 module/state-paths +8576354be3aa9ea9d4429c3ef048753a4b02c3d1771538a9c563d2504998f1ae module/status-helpers f097d0096b21c8a052f0f649b7512ecf2aba4744ae6956f001950e053828b309 module/string-coerce-runtime -c2aca425088c2bc9a74035f34d8b541354792eb0c44b988e314d59c25ac3aa7f module/telegram-account +c6ea76a9fa7f56771cbfc54617aa9ceec3d87241bab38a094622ea9351f3cc87 module/telegram-account aef35bee2502cd6ed8765409b758e452aff8ac9469fd773e6a2a44c9a1bc3f66 module/temp-path 87fa81b9e58d8fc04a4b4202d2d37fca339615f5225687d9db905151439e0f4d module/text-chunking -434804a2166f6bf2e872bff0f03f3af850e44dea04b49b1d13b41df872bb71d1 module/text-runtime -af715dd4f3083bd02dc9ce2500c670a5dd0286b84dc39a99ebd83eafc5991c74 module/tool-plugin +2ce8b180da90b5b1665bc65dd9cae9af7eb4e0988cc2a64685e8e6f9cb9a87c3 module/text-runtime +a6c3a2d8fae2811fe3aab974dbd594cd873ee049ddcd9f114df68660c5e9ed4c module/tool-plugin dc1a073c59ab61e2789533b777b3f0cb9af689d64a97796b10e8aa82552510db module/tool-results -788e35ecca74d535b95b109f6837025b97cd2b4854008166b9f2e4832c31210a module/tool-send +3c97f778d2844ba1bfd3e77fbccd3bfed94bc102053c091b00a0d3d2aaff6a99 module/tool-send cda105b721d498df23a554c6b68be150b8fe66b8b9172185c31a0b3b0646b1dc module/web-media -3b2912fca80954493f5c32c971e63135ef37596d4acde4de36c9a07a1b866dee module/webhook-ingress -a107b97d3c1bb7494e516760d613950d30dbf57ccaea4b037e2e832ca6115839 module/webhook-request-guards +740cabbc344e54f71d9a26609b20fc28a5eaa13372bd66dd40f4dfa7ffd0508a module/webhook-ingress +216e54c25ec0985fc483899d4de3d2582053a4fc3a2d1c98a8b5e7cdd2f136e1 module/webhook-request-guards de59e86e126b75d13251cba7ebbe27b44d9b5588785d98df5ff4d6722374c81f module/widget-html 9161b36ec0ab062ea41b363c894fcd672a7727f21cb726739f99f9c184fce69d module/zod diff --git a/docs/concepts/experimental-features.md b/docs/concepts/experimental-features.md index 5a95e85b7e88..cdee28e7bda8 100644 --- a/docs/concepts/experimental-features.md +++ b/docs/concepts/experimental-features.md @@ -21,6 +21,7 @@ Experimental features are preview surfaces behind explicit flags. They need more | Local model runtime | `agents.defaults.experimental.localModelLean`, `agents.entries.*.experimental.localModelLean` | A smaller or stricter local backend chokes on OpenClaw's full default tool surface | [Local Models](/gateway/local-models) | | Codex harness | `plugins.entries.codex.config.appServer.experimental.sandboxExecServer` | You want native Codex app-server 0.143.0 or newer to target an OpenClaw sandbox-backed exec-server instead of disabling Code Mode | [Codex harness reference](/plugins/codex-harness-reference#sandboxed-native-execution) | | Code Mode | `tools.codeMode.enabled` | You want compact code-orchestrated access to a hidden OpenClaw tool catalog | [Code Mode](/tools/code-mode) | +| Cloud workers | `cloudWorkers.desktop` | You want to watch or control desktop-capable cloud worker environments from the Control UI | [Cloud Worker Desktop](/gateway/cloud-workers#desktop-interactive) | | Swarm | `tools.swarm.enabled` | You want Code Mode scripts to orchestrate bounded groups of sub-agents in parallel | [Swarm](/tools/swarm) | ## Control UI Labs @@ -30,9 +31,11 @@ Control UI switch. Enabling or disabling a lab patches the canonical Gateway config immediately; the page shows a restart hint only when a feature requires one. -Code Mode and Swarm are the currently shipped Labs entries. Both switches -write existing validated config keys and normally take effect for future agent -runs without restarting the Gateway. +The currently shipped Labs entries are Code Mode, Swarm, Tool Search, +Tool-loop detection, Lean tools for local models, Message audit metadata, and +Cloud Worker Desktop. Message audit metadata and Cloud Worker Desktop require a +Gateway restart; the other switches normally take effect for future agent runs +without restarting. ## Local model lean mode diff --git a/docs/gateway/cloud-workers.md b/docs/gateway/cloud-workers.md index 15e2535adc08..b951978b27d4 100644 --- a/docs/gateway/cloud-workers.md +++ b/docs/gateway/cloud-workers.md @@ -98,11 +98,12 @@ Add a profile under `cloudWorkers.profiles` in `openclaw.json`: Profile fields: -| Key | Meaning | -| ---------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `provider` | Worker provider id registered by a plugin (`crabbox` for the bundled plugin). | -| `install` | `bundle` (default) ships the running Gateway's build; `npm` installs the exact released Gateway version with pinned integrity. `npm` requires the Gateway to run from a packaged release. | -| `settings` | Provider-owned JSON. For crabbox: `provider` (backend), `class` (machine class), `ttl`, `idleTimeout` (Go durations), optional `setup` and absolute `binary` path. OpenClaw forces public SSH and disables managed Tailscale for these leases. | +| Key | Meaning | +| ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `provider` | Worker provider id registered by a plugin (`crabbox` for the bundled plugin). | +| `install` | `bundle` (default) ships the running Gateway's build; `npm` installs the exact released Gateway version with pinned integrity. `npm` requires the Gateway to run from a packaged release. | +| `settings` | Provider-owned JSON. For crabbox: `provider` (backend), `class` (machine class), `ttl`, `idleTimeout` (Go durations), optional `setup`, optional `desktop` (boolean), and absolute `binary` path. OpenClaw forces public SSH and disables managed Tailscale for these leases. | +| `lifetime` | Optional stored policy (`idleTimeoutMinutes`, `maxLifetimeMinutes`). | Crabbox inspect reports a primary SSH port and may advertise ordered fallback ports. OpenClaw persists that order across Gateway restarts. Its shared pinned SSH transport rotates candidates only for replay-safe operations: idempotent probes, content-addressed transfers, receipt/lock-guarded artifact installation, convergent managed-worktree mirroring, and tunnel reconnects. Ambiguous unguarded stateful commands fail closed on their current candidate and are not replayed on another port. OpenClaw never invents an unadvertised port. If your network policy pins SSH ingress, allow at least one advertised Crabbox candidate. @@ -184,6 +185,18 @@ openclaw gateway call sessions.reclaim \ Placement moves through a durable state machine (`local → requested → provisioning → syncing → starting → active`), so a Gateway restart mid-dispatch reconciles instead of leaking machines. A failed model turn keeps the active placement available for a retry. Workspace path conflicts keep the local version, apply the rest of the cloud result, and preserve the staged cloud ref for inspection; other reconciliation or lifecycle failures retain their durable recovery fence and diagnostic tail until recovery can safely retry or reclaim the environment. +## Desktop (interactive) + +Cloud Worker Desktop is an experimental Labs feature and is off by default. Enable **Cloud Worker Desktop** in **Settings → Agents & Tools → Labs**, or set `cloudWorkers.desktop: true`, then restart the Gateway for the Desktop panel to appear. + +Set `"desktop": true` in a crabbox profile's `settings` to lease worker boxes with an interactive desktop (TigerVNC on the box loopback with a per-lease password). The Labs gate enables the observer surface and panel; the profile setting gives newly leased workers the desktop capability. Desktop is a warm-time capability: it cannot be added to an already-provisioned environment, so enable it on the profile before dispatching. + +Operators with `operator.admin` access watch and control the desktop from the Control UI **Desktop** panel (also in the command palette). The panel lists desktop-capable environments from `environments.list` and connects through the Gateway, which forwards the box's loopback VNC over the same pinned SSH transport used for worker traffic — the desktop is never exposed on the box's network, and the VNC password is delivered only inside the authenticated `worker.desktop.observe` RPC result, never stored by the Gateway. + +Connections start view-only. **Take control** requests an input-capable connection; only one controller is active at a time, and taking control disconnects the previous controller (they are downgraded to view-only). Up to 8 observers can watch one environment. The desktop forward starts on first observe and shuts down about a minute after the last observer disconnects; stopping or reclaiming the environment tears it down immediately. + +Desktop observe is not supported when the Gateway itself runs on Windows. + ## Security model - **Closed worker ingress.** Workers speak a dedicated protocol on the tunneled socket with a closed method allowlist — a worker cannot call operator RPCs. diff --git a/docs/gateway/protocol.md b/docs/gateway/protocol.md index 5dfd7151ee43..07b672605cf7 100644 --- a/docs/gateway/protocol.md +++ b/docs/gateway/protocol.md @@ -625,6 +625,7 @@ methods. Treat this as feature discovery, not a full enumeration of - `artifacts.list`, `artifacts.get`, and `artifacts.download` expose transcript-derived artifact summaries and downloads for an explicit `sessionKey`, `runId`, or `taskId` scope. Run and task queries resolve the owning session server-side and only return transcript media with matching provenance; unsafe or local URL sources return unsupported downloads instead of fetching server-side. - `environments.list` and `environments.status` remain available without cloud-worker profiles and preserve gateway-local and node environment discovery. Configured cloud workers and durable records left by earlier profiles add `worker` metadata with `providerId`, optional `leaseId`, `state`, `ageMs`, optional `idleMs`, and `attachedSessionIds`. Worker lifecycle states are `requested`, `provisioning`, `bootstrapping`, `ready`, `attached`, `idle`, `draining`, `destroying`, `destroyed`, `failed`, and `orphaned`. - `environments.create` (`{ profileId, idempotencyKey }`) provisions a worker from a configured plugin provider profile; retries with the same key reuse the durable operation. `environments.destroy` (`{ environmentId }`) requests idempotent teardown of a durable worker environment. Both require `operator.admin`, are control-plane writes, and return the same environment summary shape used by status responses. + - `worker.desktop.observe` (`{ environmentId, control? }`, `operator.admin`) starts or reuses the environment's desktop forward and returns `{ transport, wsPath, expiresAtMs, control, vncPassword? }`. `wsPath` carries a single-use 60-second token for the Gateway's desktop observer WebSocket; reconnecting requires a fresh observe call. Environments with an observable desktop advertise `worker.desktop: true` in `environments.list`. The method is advertised only when the `cloudWorkers.desktop` lab is enabled. See [Cloud workers](/gateway/cloud-workers#desktop-interactive). - `agent.identity.get` returns the effective assistant identity for an agent or session. - `agent.wait` waits for a run to finish and returns the terminal snapshot when available. diff --git a/docs/plugins/sdk-overview.md b/docs/plugins/sdk-overview.md index dd2e890b29ec..97d1557c21cb 100644 --- a/docs/plugins/sdk-overview.md +++ b/docs/plugins/sdk-overview.md @@ -120,6 +120,7 @@ and external URLs. Registering another provider replaces the current provider. Worker providers must also declare their id in `contracts.workerProviders`. Core persists durable intent before `provision(profile, operationId)`. Providers validate settings before external allocation and throw `WorkerProviderError` for permanent profile rejection. `provision` must adopt the same lease when the operation id repeats. Core persists the validated profile settings with the lease and supplies that snapshot to `destroy({ leaseId, profile })`, which must be idempotent, and `inspect({ leaseId, profile })`, which returns `active`, `destroyed`, or `unknown`. This lets providers route lifecycle calls after a gateway restart or named-profile removal. SSH endpoints use a `SecretRef` for `keyRef`, never inline key material, and include a `hostKey` from trusted provisioning output as exactly `algorithm base64`, without a hostname or comment. Core pins `hostKey` and never trusts a key from the first connection. Providers may also return up to 10 ordered, unique `fallbackPorts` (integer ports from 1 through 65535, excluding the primary `port`); core validates and persists those advertised candidates for idempotent probes, content-addressed transfers, receipt/lock-guarded artifact installation, convergent managed-worktree mirroring, and tunnel reconnects. Ambiguous unguarded stateful commands fail closed and are not replayed across candidates. A lease may set `sharedHost: true` when the SSH account also owns unrelated processes; core then avoids host-wide process freezing during workspace reconciliation. Omitted or `false` means a dedicated worker host. Active inspection repeats this fact so core can reconcile provider-owned isolation for leases persisted before the field existed; tunnel startup waits for that first authoritative inspection. A provider that mints a dynamic `keyRef` can implement `resolveSshIdentity({ leaseId, profile, keyRef })`; when present, that resolver is authoritative, while providers without it use the configured generic secret resolver. +`WorkerLease.desktop` is optional and has the shape `{ protocol: "rfb"; port: number; passwordFilePath?: string }`; `passwordFilePath`, when present, must be absolute. Providers report this warm-time capability from `provision`; it cannot be retrofitted onto a live lease. The Gateway reads the password file over the provider's SSH endpoint when needed and never persists the password. Providers with renewable leases can also implement `renew(leaseId)`. `inspect` must throw on transient or indeterminate failures; return `unknown` only for authoritative absence. Core marks an active local record orphaned, or treats the absence as teardown completion after a persisted destroy request. diff --git a/extensions/crabbox/src/crabbox-worker-profile.ts b/extensions/crabbox/src/crabbox-worker-profile.ts index 0ee85350a756..77bf00f5fb1e 100644 --- a/extensions/crabbox/src/crabbox-worker-profile.ts +++ b/extensions/crabbox/src/crabbox-worker-profile.ts @@ -3,7 +3,15 @@ import fs from "node:fs"; import path from "node:path"; import { WorkerProviderError, type WorkerProfile } from "openclaw/plugin-sdk/plugin-entry"; -const PROFILE_KEYS = new Set(["binary", "class", "idleTimeout", "provider", "setup", "ttl"]); +const PROFILE_KEYS = new Set([ + "binary", + "class", + "desktop", + "idleTimeout", + "provider", + "setup", + "ttl", +]); const GO_DURATION_PATTERN = /^\+?(?:(?:\d+(?:\.\d*)?|\.\d+)(?:ns|us|µs|μs|ms|s|m|h))+$/u; const GO_DURATION_TOKEN_PATTERN = /(\d+(?:\.\d*)?|\.\d+)(ns|us|µs|μs|ms|s|m|h)/gu; const MAX_GO_DURATION_NANOSECONDS = 9_223_372_036_854_775_807n; @@ -21,6 +29,7 @@ const DURATION_UNIT_NANOSECONDS: Readonly> = { type CrabboxProfile = { binary?: string; class: string; + desktop?: boolean; idleTimeout: string; provider: string; ttl: string; @@ -105,7 +114,35 @@ export function parseCrabboxProfile(profile: WorkerProfile): CrabboxProfile { if (setupValue !== undefined && !setup) { throw new WorkerProviderError("Crabbox profile setup must be a non-empty command string"); } - return { binary, class: machineClass, idleTimeout, provider, setup, ttl }; + const desktop = profile.desktop; + if (desktop !== undefined && typeof desktop !== "boolean") { + throw new WorkerProviderError("Crabbox profile desktop must be a boolean"); + } + return { binary, class: machineClass, desktop, idleTimeout, provider, setup, ttl }; +} + +export function buildCrabboxWarmupArgs(profile: CrabboxProfile, slug: string): string[] { + const args = [ + "warmup", + "--provider", + profile.provider, + "--network", + "public", + "--tailscale=false", + "--class", + profile.class, + "--ttl", + profile.ttl, + "--idle-timeout", + profile.idleTimeout, + "--slug", + slug, + "--keep=true", + ]; + if (profile.desktop) { + args.push("--desktop"); + } + return args; } function defaultIsExecutable(candidate: string, platform: NodeJS.Platform): boolean { diff --git a/extensions/crabbox/src/crabbox-worker-provider.test.ts b/extensions/crabbox/src/crabbox-worker-provider.test.ts index a4da00d54d1d..c91456982765 100644 --- a/extensions/crabbox/src/crabbox-worker-provider.test.ts +++ b/extensions/crabbox/src/crabbox-worker-provider.test.ts @@ -656,6 +656,52 @@ describe("Crabbox worker provider", () => { ); }); + it("rejects a non-boolean desktop profile setting", async () => { + const provider = providerWithRunner(async () => commandResult()); + await expect( + provider.provision({ ...PROFILE, desktop: "yes" }, "provision:desktop-invalid"), + ).rejects.toThrow("Crabbox profile desktop must be a boolean"); + }); + + it("adds --desktop only for desktop warmups and returns the endpoint", async () => { + const calls: string[][] = []; + let warmed = false; + const provider = providerWithRunner(async (argv) => { + calls.push(argv); + if (argv[1] === "warmup") { + warmed = true; + return commandResult({ stdout: `leased ${LEASE_ID} slug=test\n` }); + } + return warmed || argv.includes(LEASE_ID) + ? commandResult({ stdout: inspectJson({ sshHostKey: HOST_KEY }) }) + : commandResult({ code: 4, stderr: `lease/server not found: ${argv.at(-2)}` }); + }); + + await expect( + provider.provision({ ...PROFILE, desktop: true }, "provision:desktop-fresh"), + ).resolves.toMatchObject({ + desktop: { + protocol: "rfb", + port: 5900, + passwordFilePath: "/var/lib/crabbox/vnc.password", + }, + }); + expect(calls.find((argv) => argv[1] === "warmup")).toContain("--desktop"); + }); + + it("adopts desktop metadata on replay without another warmup", async () => { + const calls: string[][] = []; + const provider = providerWithRunner(async (argv) => { + calls.push(argv); + return commandResult({ stdout: inspectJson({ sshHostKey: HOST_KEY }) }); + }); + + await expect( + provider.provision({ ...PROFILE, desktop: true }, "provision:desktop-replay"), + ).resolves.toMatchObject({ desktop: { protocol: "rfb", port: 5900 } }); + expect(calls.some((argv) => argv[1] === "warmup")).toBe(false); + }); + it("stops a newly provisioned lease when inspect cannot supply a host key", async () => { const calls: Array<{ argv: string[]; options: Parameters[1] }> = []; const runCommand: CrabboxCommandRunner = async (argv, options) => { diff --git a/extensions/crabbox/src/crabbox-worker-provider.ts b/extensions/crabbox/src/crabbox-worker-provider.ts index 734973b1a35b..3f111c8cad0a 100644 --- a/extensions/crabbox/src/crabbox-worker-provider.ts +++ b/extensions/crabbox/src/crabbox-worker-provider.ts @@ -11,6 +11,7 @@ import { runCommandWithTimeout, type SpawnResult } from "openclaw/plugin-sdk/pro import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime"; import { parseInspectJson, type ParsedInspect } from "./crabbox-worker-inspect.js"; import { + buildCrabboxWarmupArgs, identityRefId, nonEmptyString, operationSlug, @@ -53,6 +54,7 @@ const LEASE_ID_PATTERN = /^(?:cbx_|tbx_)[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/u; const LEASE_TOKEN_IN_OUTPUT_PATTERN = /^leased\s+(\S{1,128})(?=\s|$)/mu; type CrabboxCommandRunner = typeof runCommandWithTimeout; +type CrabboxProfile = ReturnType; type LeaseCommandContext = { binary: string; @@ -64,6 +66,7 @@ type ProvisionInspectContext = { binary: string; deadline: number; inspect: ParsedInspect; + profile: CrabboxProfile; provider: string; runCommand: CrabboxCommandRunner; }; @@ -336,7 +339,7 @@ function statusFromInspect(inspect: ParsedInspect): WorkerLeaseStatus { return { status: "active" }; } -function leaseFromInspect(inspect: ParsedInspect): WorkerLease { +function leaseFromInspect(inspect: ParsedInspect, profile: CrabboxProfile): WorkerLease { if (isTerminalState(inspect.state)) { throw new Error("Crabbox operation lease is no longer active"); } @@ -367,13 +370,24 @@ function leaseFromInspect(inspect: ParsedInspect): WorkerLease { id: identityRefId(inspect.id), }, }, + // Crabbox's Linux desktop contract is TigerVNC on worker loopback with a per-lease + // password file. This warm-time capability cannot be retrofitted onto an existing lease. + ...(profile.desktop + ? { + desktop: { + protocol: "rfb" as const, + port: 5900, + passwordFilePath: "/var/lib/crabbox/vnc.password", + }, + } + : {}), }; } async function leaseFromProvisionInspect(params: ProvisionInspectContext): Promise { try { assertProvisionSecurityPolicy(params); - return leaseFromInspect(params.inspect); + return leaseFromInspect(params.inspect, params.profile); } catch (error) { await stopProvisionInspect(params); throw error; @@ -589,6 +603,7 @@ export function createCrabboxWorkerProvider( binary, deadline, inspect: existing.inspect, + profile: parsed, provider: parsed.provider, runCommand, }); @@ -601,6 +616,7 @@ export function createCrabboxWorkerProvider( binary, deadline, inspect: existing.inspect, + profile: parsed, provider: parsed.provider, runCommand, }; @@ -628,23 +644,7 @@ export function createCrabboxWorkerProvider( const warmup = await runCrabboxCommand({ action: "warmup", - args: [ - "warmup", - "--provider", - parsed.provider, - "--network", - "public", - "--tailscale=false", - "--class", - parsed.class, - "--ttl", - parsed.ttl, - "--idle-timeout", - parsed.idleTimeout, - "--slug", - slug, - "--keep=true", - ], + args: buildCrabboxWarmupArgs(parsed, slug), binary, runCommand, timeoutMs: remainingProvisionTimeout(deadline, WARMUP_TIMEOUT_MS), @@ -695,6 +695,7 @@ export function createCrabboxWorkerProvider( binary, deadline, inspect: inspected.inspect, + profile: parsed, provider: parsed.provider, runCommand, }; diff --git a/packages/gateway-protocol/src/index.test.ts b/packages/gateway-protocol/src/index.test.ts index 4843ef17ddab..c89c01b92608 100644 --- a/packages/gateway-protocol/src/index.test.ts +++ b/packages/gateway-protocol/src/index.test.ts @@ -487,6 +487,47 @@ describe("lazy protocol validators", () => { ]); }); + it("validates worker desktop observer request and result contracts", () => { + expectAccepted(protocol.validateWorkerDesktopObserveParams, [ + { environmentId: "worker:one" }, + { environmentId: "worker:one", control: true }, + ]); + expectRejected(protocol.validateWorkerDesktopObserveParams, [ + { environmentId: "" }, + { environmentId: "worker:one", control: "yes" }, + { environmentId: "worker:one", extra: true }, + ]); + expectAccepted(protocol.validateWorkerDesktopObserveResult, [ + { + transport: "rfb", + wsPath: "/worker-desktop/observe?token=abc", + expiresAtMs: 60_000, + control: false, + }, + { + transport: "rfb", + wsPath: "/worker-desktop/observe?token=abc", + expiresAtMs: 60_000, + control: true, + vncPassword: "secret", + }, + ]); + expectRejected(protocol.validateWorkerDesktopObserveResult, [ + { + transport: "vnc", + wsPath: "/worker-desktop/observe?token=abc", + expiresAtMs: 60_000, + control: false, + }, + { + transport: "rfb", + wsPath: "", + expiresAtMs: -1, + control: false, + }, + ]); + }); + it("validates chat sends that suppress command interpretation", () => { expectAccepted(validateChatSendParams, [ { diff --git a/packages/gateway-protocol/src/schema-export-registry.ts b/packages/gateway-protocol/src/schema-export-registry.ts index 35b2e273fcaf..e22e297052d7 100644 --- a/packages/gateway-protocol/src/schema-export-registry.ts +++ b/packages/gateway-protocol/src/schema-export-registry.ts @@ -79,6 +79,8 @@ export { EnvironmentsListResultSchema, EnvironmentsStatusParamsSchema, EnvironmentsStatusResultSchema, + WorkerDesktopObserveParamsSchema, + WorkerDesktopObserveResultSchema, SystemInfoParamsSchema, SystemInfoResultSchema, StateVersionSchema, diff --git a/packages/gateway-protocol/src/schema/environments.ts b/packages/gateway-protocol/src/schema/environments.ts index be8987118eb5..d4bebc66c91c 100644 --- a/packages/gateway-protocol/src/schema/environments.ts +++ b/packages/gateway-protocol/src/schema/environments.ts @@ -47,6 +47,7 @@ export const WorkerEnvironmentMetadataSchema = closedObject({ attachedSessionIds: Type.Array(NonEmptyString), tunnelStatus: WorkerTunnelStatusSchema, error: Type.Optional(NonEmptyString), + desktop: Type.Optional(Type.Boolean()), }); function createEnvironmentSummarySchema() { @@ -102,6 +103,21 @@ export const EnvironmentsDestroyParamsSchema = closedObject({ /** Destroy result exposes the terminal worker lifecycle state. */ export const EnvironmentsDestroyResultSchema = createEnvironmentSummarySchema(); +export const WorkerDesktopObserveParamsSchema = closedObject({ + environmentId: NonEmptyString, + control: Type.Optional(Type.Boolean()), +}); + +// Transport is an open enum-string; future transports may add split streamPath/controlPath +// fields additively without replacing the phase-1 RFB contract. +export const WorkerDesktopObserveResultSchema = closedObject({ + transport: Type.String({ enum: ["rfb"] }), + wsPath: NonEmptyString, + expiresAtMs: Type.Integer({ minimum: 0 }), + control: Type.Boolean(), + vncPassword: Type.Optional(NonEmptyString), +}); + export type EnvironmentStatus = Static; export type WorkerEnvironmentState = Static; export type WorkerTunnelStatus = Static; @@ -115,3 +131,5 @@ export type EnvironmentsListParams = Static export type EnvironmentsListResult = Static; export type EnvironmentsStatusParams = Static; export type EnvironmentsStatusResult = Static; +export type WorkerDesktopObserveParams = Static; +export type WorkerDesktopObserveResult = Static; diff --git a/packages/gateway-protocol/src/schema/protocol-schema-fragment-agent-control.ts b/packages/gateway-protocol/src/schema/protocol-schema-fragment-agent-control.ts index 38ef08b4adda..d76a46e65fff 100644 --- a/packages/gateway-protocol/src/schema/protocol-schema-fragment-agent-control.ts +++ b/packages/gateway-protocol/src/schema/protocol-schema-fragment-agent-control.ts @@ -18,6 +18,8 @@ export const AgentControlProtocolSchemas = { EnvironmentsListResult: environments.EnvironmentsListResultSchema, EnvironmentsStatusParams: environments.EnvironmentsStatusParamsSchema, EnvironmentsStatusResult: environments.EnvironmentsStatusResultSchema, + WorkerDesktopObserveParams: environments.WorkerDesktopObserveParamsSchema, + WorkerDesktopObserveResult: environments.WorkerDesktopObserveResultSchema, SystemInfoParams: systemInfo.SystemInfoParamsSchema, SystemInfoResult: systemInfo.SystemInfoResultSchema, AgentEvent: agent.AgentEventSchema, diff --git a/packages/gateway-protocol/src/validator-registry.ts b/packages/gateway-protocol/src/validator-registry.ts index c378a70378ff..267dc021371e 100644 --- a/packages/gateway-protocol/src/validator-registry.ts +++ b/packages/gateway-protocol/src/validator-registry.ts @@ -144,6 +144,8 @@ export const validateEnvironmentsCreateParams = compile(S.EnvironmentsCreatePara export const validateEnvironmentsDestroyParams = compile(S.EnvironmentsDestroyParamsSchema); export const validateEnvironmentsListParams = compile(S.EnvironmentsListParamsSchema); export const validateEnvironmentsStatusParams = compile(S.EnvironmentsStatusParamsSchema); +export const validateWorkerDesktopObserveParams = compile(S.WorkerDesktopObserveParamsSchema); +export const validateWorkerDesktopObserveResult = compile(S.WorkerDesktopObserveResultSchema); export const validateSystemInfoParams = compile(S.SystemInfoParamsSchema); export const validateSystemInfoResult = compile(S.SystemInfoResultSchema); export const validateNodePendingAckParams = compile(S.NodePendingAckParamsSchema); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ef99b5ac0299..2c81f8a31895 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2407,6 +2407,9 @@ importers: '@noble/ed25519': specifier: 3.1.0 version: 3.1.0 + '@novnc/novnc': + specifier: ^1.7.0 + version: 1.7.0 '@openclaw/gateway-client': specifier: workspace:* version: link:../packages/gateway-client @@ -4077,6 +4080,9 @@ packages: resolution: {integrity: sha512-tlc/FcYIv5i8RYsl2iDil4A0gOihaas1R5jPcIC4Zw3GhjKsVilw90aHcVlhZPTBLGBzd379S+VcnsDjd9ChiA==} engines: {node: '>=12.4.0'} + '@novnc/novnc@1.7.0': + resolution: {integrity: sha512-ucEJOx4T2avIRCleodk7YobZj5O2Ga2AeLfQ69A/yjG9HHba2+PDgwSkN3FttrmG+70ZGx21sElNFouK13RzyA==} + '@npmcli/agent@5.0.2': resolution: {integrity: sha512-EkzGmEsgbQ1rqWkRJe2P0oQHx/ylZozDUNPMXCklLuSFL3GY+QyEfBUjhjCsgGXzh4OGpnHvkboSQgczjP/jJg==} engines: {node: ^22.22.2 || ^24.15.0 || >=26.0.0} @@ -11364,6 +11370,8 @@ snapshots: '@nolyfill/domexception@1.0.28': {} + '@novnc/novnc@1.7.0': {} + '@npmcli/agent@5.0.2(supports-color@10.2.2)': dependencies: agent-base: 9.0.0 diff --git a/scripts/plugin-sdk-surface-report.mts b/scripts/plugin-sdk-surface-report.mts index 8a335c051eb0..f3beb0a536b8 100644 --- a/scripts/plugin-sdk-surface-report.mts +++ b/scripts/plugin-sdk-surface-report.mts @@ -249,7 +249,8 @@ export function readPluginSdkSurfaceBudgets(env: NodeJS.ProcessEnv = process.env // +1: shared transcript credential-safety prompt for plugin-owned agent harnesses. // +3: channel streaming config reader re-exports and session-agent scope resolver. // +3: session-catalog terminal-start provider request and Gateway params/result contracts. - 4847, + // +1: worker desktop endpoint contract for desktop-capable worker leases. + 4848, env, ), publicFunctionExports: readPluginSdkSurfaceBudgetEnv( diff --git a/scripts/protocol-gen-kotlin.ts b/scripts/protocol-gen-kotlin.ts index 5d92abc31929..d6c150c74e91 100644 --- a/scripts/protocol-gen-kotlin.ts +++ b/scripts/protocol-gen-kotlin.ts @@ -56,6 +56,8 @@ const schemaNames = new Map([ ["QuestionListResult", "QuestionListResult"], ["SessionObserverPlanProgress", "SessionObserverPlanProgress"], ["SessionObserverDigest", "SessionObserverDigest"], + ["WorkerDesktopObserveParams", "WorkerDesktopObserveParams"], + ["WorkerDesktopObserveResult", "WorkerDesktopObserveResult"], ]); const androidEnums: EnumSpec[] = [ diff --git a/src/config/types.cloud-workers.ts b/src/config/types.cloud-workers.ts index efd8aac00a6d..6a594c5498ea 100644 --- a/src/config/types.cloud-workers.ts +++ b/src/config/types.cloud-workers.ts @@ -10,6 +10,8 @@ export type CloudWorkerProfileConfig = { }; export type CloudWorkersConfig = { + /** Experimental Labs gate for the cloud-worker desktop observer. */ + desktop?: boolean; /** Named opt-in worker profiles. Omit or leave empty to disable cloud workers. */ profiles?: Record; }; diff --git a/src/config/zod-schema.cloud-workers.test.ts b/src/config/zod-schema.cloud-workers.test.ts index b79426e9eca2..28a7c01333c0 100644 --- a/src/config/zod-schema.cloud-workers.test.ts +++ b/src/config/zod-schema.cloud-workers.test.ts @@ -22,6 +22,10 @@ describe("OpenClawSchema cloudWorkers config", () => { title?: string; description?: string; properties?: { + desktop?: { + title?: string; + description?: string; + }; profiles?: { title?: string; description?: string; @@ -36,10 +40,12 @@ describe("OpenClawSchema cloudWorkers config", () => { }; } ).properties?.cloudWorkers; + const desktop = properties?.properties?.desktop; const profiles = properties?.properties?.profiles; const profile = profiles?.additionalProperties; for (const [path, schema] of [ + ["cloudWorkers.desktop", desktop], ["cloudWorkers.profiles", profiles], ["cloudWorkers.profiles.*", profile], ["cloudWorkers.profiles.*.provider", profile?.properties?.provider], @@ -58,6 +64,11 @@ describe("OpenClawSchema cloudWorkers config", () => { expect(parseCloudWorkers({})).toStrictEqual({}); }); + it("accepts the desktop Labs gate only as a boolean", () => { + expect(parseCloudWorkers({ desktop: true })).toStrictEqual({ desktop: true }); + expect(OpenClawSchema.safeParse({ cloudWorkers: { desktop: "true" } }).success).toBe(false); + }); + it("accepts provider-owned settings", () => { expect( parseCloudWorkers({ diff --git a/src/config/zod-schema.cloud-workers.ts b/src/config/zod-schema.cloud-workers.ts index 82a7aab87a11..8741c854487c 100644 --- a/src/config/zod-schema.cloud-workers.ts +++ b/src/config/zod-schema.cloud-workers.ts @@ -87,6 +87,10 @@ const CloudWorkerProfileIdSchema = z ); const CloudWorkersConfigShape = { + desktop: z.boolean().optional().register(configUiMetadata, { + label: "Cloud Worker Desktop (Labs)", + help: "Enables the experimental worker.desktop.observe surface and Control UI Desktop panel for desktop-capable cloud worker environments.", + }), profiles: z .record(CloudWorkerProfileIdSchema, CloudWorkerProfileSchema) .optional() @@ -99,6 +103,7 @@ const CloudWorkersConfigShape = { export const CloudWorkersConfigSchema = z.object(CloudWorkersConfigShape).strict().optional(); const CLOUD_WORKER_FIELD_SCHEMAS = { + "cloudWorkers.desktop": CloudWorkersConfigShape.desktop, "cloudWorkers.profiles": CloudWorkersConfigShape.profiles, "cloudWorkers.profiles.*": CloudWorkerProfileSchema, "cloudWorkers.profiles.*.provider": CloudWorkerProfileShape.provider, diff --git a/src/gateway/methods/core-descriptors.since.test.ts b/src/gateway/methods/core-descriptors.since.test.ts index 9c1fb043f896..7e5f4ae5af81 100644 --- a/src/gateway/methods/core-descriptors.since.test.ts +++ b/src/gateway/methods/core-descriptors.since.test.ts @@ -99,5 +99,8 @@ describe("core gateway method release trains", () => { expect(methods.find((method) => method.name === "sessions.catalog.startTerminal")?.since).toBe( "2026.8", ); + expect(methods.find((method) => method.name === "worker.desktop.observe")?.since).toBe( + "2026.8", + ); }); }); diff --git a/src/gateway/methods/core-descriptors.ts b/src/gateway/methods/core-descriptors.ts index dead84830982..1fdd612ed0ba 100644 --- a/src/gateway/methods/core-descriptors.ts +++ b/src/gateway/methods/core-descriptors.ts @@ -491,6 +491,7 @@ const CORE_GATEWAY_METHOD_SPECS = [ ["update.hold", "update", "operator.admin", "2026.8", { controlPlaneWrite: true }], // Additive catalog terminal start appends so older advertised indices stay stable. ["sessions.catalog.startTerminal", "session-catalog", "operator.admin", "2026.8"], + ["worker.desktop.observe", "environments", "operator.admin", "2026.8", { startup: true }], ] as const satisfies readonly CoreGatewayMethodSpecRow[]; export type CoreGatewayHandlerFamily = Exclude<(typeof CORE_GATEWAY_METHOD_SPECS)[number][1], null>; diff --git a/src/gateway/server-core-runtime.ts b/src/gateway/server-core-runtime.ts index 40e3034a386f..848209c5a802 100644 --- a/src/gateway/server-core-runtime.ts +++ b/src/gateway/server-core-runtime.ts @@ -133,6 +133,7 @@ export async function startGatewayCoreRuntime(input: { workerEnvironmentService, workerPlacementDispatchAvailable, workerPlacementControlAvailable, + workerDesktopObserveAvailable, listStartupChannelGatewayMethods, coreGatewayMethodNames, pluginHostServices, @@ -314,7 +315,8 @@ export async function startGatewayCoreRuntime(input: { (descriptor.name !== "environments.create" && descriptor.name !== "environments.destroy")) && (workerPlacementDispatchAvailable || descriptor.name !== "sessions.dispatch") && - (workerPlacementControlAvailable || descriptor.name !== "sessions.reclaim"), + (workerPlacementControlAvailable || descriptor.name !== "sessions.reclaim") && + (workerDesktopObserveAvailable || descriptor.name !== "worker.desktop.observe"), ); return createGatewayMethodRegistry( [ diff --git a/src/gateway/server-http.ts b/src/gateway/server-http.ts index e4b2de54d0b3..1ad86491b3b5 100644 --- a/src/gateway/server-http.ts +++ b/src/gateway/server-http.ts @@ -73,6 +73,7 @@ import { } from "./server/ws-types.js"; import { isTerminalConfigEnabled } from "./terminal/enabled.js"; import { canonicalizeUserProfileAvatarPath } from "./user-profiles-http-path.js"; +import type { WorkerDesktopTunnels } from "./worker-environments/desktop-tunnel.js"; type PluginGatewayDispatchContext = { gatewayAuthSatisfied?: boolean; @@ -813,6 +814,7 @@ export function attachGatewayUpgradeHandler(opts: { rateLimiter?: AuthRateLimiter; /** Optional logger for error diagnostics. */ log?: { warn: (msg: string) => void }; + workerDesktopTunnels?: WorkerDesktopTunnels; }) { const { httpServer, @@ -912,6 +914,27 @@ export function attachGatewayUpgradeHandler(opts: { return; } } + if (requestPath === "/worker-desktop/observe") { + if (!opts.workerDesktopTunnels) { + writeGatewayUpgradeServiceUnavailable(socket, "desktop observe unavailable"); + socket.destroy(); + return; + } + // Desktop observers are long-lived Gateway sockets, so they obey the same + // suspension/restart admission boundary as core upgrades. Without this a + // drained Gateway would keep accepting new desktop streams. + if (isGatewayWorkAdmissionClosed()) { + writeGatewayUpgradeServiceUnavailable(socket, "Gateway websocket admission closed"); + socket.destroy(); + return; + } + const { handleWorkerDesktopUpgrade } = + await import("./worker-environments/desktop-observe.js"); + handleWorkerDesktopUpgrade(req, socket, head, { + tunnels: opts.workerDesktopTunnels, + }); + return; + } // Plugin-owned upgrade routes have already had the opportunity to claim the socket. // Core Gateway upgrades must stop at the HTTP boundary so a client cannot hold an // untracked pre-connect socket after suspension or restart admission closes. diff --git a/src/gateway/server-methods-list.test.ts b/src/gateway/server-methods-list.test.ts index a6ac136eb80c..60e6bdb94036 100644 --- a/src/gateway/server-methods-list.test.ts +++ b/src/gateway/server-methods-list.test.ts @@ -66,7 +66,7 @@ describe("listGatewayMethods", () => { }); it("appends new methods after model probing without shifting older method indices", () => { - expect(listGatewayMethods().slice(-34)).toEqual([ + expect(listGatewayMethods().slice(-35)).toEqual([ "models.probe", "migrations.memory.plan", "migrations.memory.apply", @@ -101,6 +101,7 @@ describe("listGatewayMethods", () => { "sessions.patchMany", "update.hold", "sessions.catalog.startTerminal", + "worker.desktop.observe", ]); const methods = listGatewayMethods(); expect(methods.indexOf("node.pluginSurface.refresh")).toBe( @@ -173,7 +174,7 @@ describe("listGatewayMethods", () => { "exec.approval.get", ]); expect(methods).toContain("tts.speak"); - expect(coreMethods.slice(-41)).toEqual([ + expect(coreMethods.slice(-42)).toEqual([ "sessions.catalog.continue", "sessions.catalog.archive", "approval.get", @@ -215,6 +216,7 @@ describe("listGatewayMethods", () => { "sessions.patchMany", "update.hold", "sessions.catalog.startTerminal", + "worker.desktop.observe", ]); expect(methods.indexOf("approval.get")).toBeGreaterThan(methods.indexOf("tts.speak")); expect(methods.indexOf("approval.resolve")).toBe(methods.indexOf("approval.get") + 1); @@ -224,6 +226,9 @@ describe("listGatewayMethods", () => { expect(methods.indexOf("sessions.catalog.startTerminal")).toBe( methods.indexOf("update.hold") + 1, ); + expect(methods.indexOf("worker.desktop.observe")).toBe( + methods.indexOf("sessions.catalog.startTerminal") + 1, + ); }); it("advertises the versioned Talk session RPCs", () => { diff --git a/src/gateway/server-methods/environments.test.ts b/src/gateway/server-methods/environments.test.ts index 1c2e96c68bb1..1f92b15267fe 100644 --- a/src/gateway/server-methods/environments.test.ts +++ b/src/gateway/server-methods/environments.test.ts @@ -21,6 +21,7 @@ vi.mock("../../infra/device-pairing-node.js", () => ({ const NOW = 10_000; type TestWorkerRecord = WorkerEnvironmentRecord & { + desktopAvailable: boolean; tunnelStatus: WorkerTunnelStatus; error?: string; }; @@ -31,6 +32,13 @@ type TestWorkerService = { create: (profileId: string, idempotencyKey: string) => Promise; destroy: (environmentId: string) => Promise; destroyUnattached: (environmentId: string) => Promise; + observeDesktop: (request: { environmentId: string; control: boolean }) => Promise<{ + transport: "rfb"; + wsPath: string; + expiresAtMs: number; + control: boolean; + vncPassword?: string; + }>; }; function mockContext( @@ -87,6 +95,7 @@ function workerRecord(overrides: Partial = {}): TestWorkerReco profileSnapshot: { settings: {} }, provisionOperationId: "provision:worker-1", leaseId: "lease-1", + desktop: null, sshEndpoint: { host: "worker.example.test", port: 22, @@ -102,6 +111,7 @@ function workerRecord(overrides: Partial = {}): TestWorkerReco idleSinceAtMs: null, lastError: null, tunnelStatus: "stopped", + desktopAvailable: false, ...overrides, } as TestWorkerRecord; } @@ -113,6 +123,12 @@ function workerService(overrides: Partial = {}) { create: vi.fn(async () => workerRecord()), destroy: vi.fn(async () => workerRecord({ state: "destroyed" })), destroyUnattached: vi.fn(async () => workerRecord({ state: "destroyed" })), + observeDesktop: vi.fn(async ({ control }) => ({ + transport: "rfb" as const, + wsPath: "/worker-desktop/observe?token=abc", + expiresAtMs: 70_000, + control, + })), ...overrides, }; } @@ -122,7 +138,8 @@ async function callEnvironmentMethod( | "environments.list" | "environments.status" | "environments.create" - | "environments.destroy", + | "environments.destroy" + | "worker.desktop.observe", params: unknown, options: { service?: TestWorkerService; @@ -276,6 +293,15 @@ describe("environment gateway methods", () => { ).not.toHaveProperty("error"); }); + it("projects desktop metadata only when the service reports it available", () => { + expect( + summarizeWorkerEnvironment(workerRecord({ desktopAvailable: true }), NOW).worker, + ).toMatchObject({ desktop: true }); + expect( + summarizeWorkerEnvironment(workerRecord({ desktopAvailable: false }), NOW).worker, + ).not.toHaveProperty("desktop"); + }); + it("returns status for one node environment", async () => { const [ok, payload] = await callEnvironmentMethod("environments.status", { environmentId: "node:node-live", @@ -434,6 +460,80 @@ describe("environment gateway methods", () => { }); }); + it("starts desktop observation with explicit and default control modes", async () => { + const observeDesktop = vi.fn(async ({ control }: { control: boolean }) => ({ + transport: "rfb" as const, + wsPath: "/worker-desktop/observe?token=abc", + expiresAtMs: 70_000, + control, + })); + const service = workerService({ observeDesktop }); + const first = await callEnvironmentMethod( + "worker.desktop.observe", + { environmentId: "worker-1", control: true }, + { service }, + ); + const second = await callEnvironmentMethod( + "worker.desktop.observe", + { environmentId: "worker-1" }, + { service }, + ); + expect(first).toEqual([ + true, + { + transport: "rfb", + wsPath: "/worker-desktop/observe?token=abc", + expiresAtMs: 70_000, + control: true, + }, + undefined, + ]); + expect(second[1]).toMatchObject({ control: false }); + expect(observeDesktop).toHaveBeenNthCalledWith(1, { + environmentId: "worker-1", + control: true, + }); + expect(observeDesktop).toHaveBeenNthCalledWith(2, { + environmentId: "worker-1", + control: false, + }); + }); + + it("maps desktop lifecycle errors to invalid request and hides runtime failures", async () => { + const invalidService = workerService({ + observeDesktop: vi.fn(async () => { + throw new FakeWorkerServiceError("invalid_state", "environment has no desktop"); + }), + }); + const unavailableService = workerService({ + observeDesktop: vi.fn(async () => { + throw new FakeWorkerServiceError("provider_failure", "private SSH failure"); + }), + }); + expect( + await callEnvironmentMethod( + "worker.desktop.observe", + { environmentId: "worker-1" }, + { service: invalidService }, + ), + ).toEqual([ + false, + undefined, + { code: ErrorCodes.INVALID_REQUEST, message: "environment has no desktop" }, + ]); + expect( + await callEnvironmentMethod( + "worker.desktop.observe", + { environmentId: "worker-1" }, + { service: unavailableService }, + ), + ).toEqual([ + false, + undefined, + { code: ErrorCodes.UNAVAILABLE, message: "worker desktop observe unavailable" }, + ]); + }); + it("destroys an environment idempotently", async () => { const destroyed = workerRecord({ state: "destroyed" }); const destroyUnattached = vi.fn(async () => destroyed); diff --git a/src/gateway/server-methods/environments.ts b/src/gateway/server-methods/environments.ts index 5c905ee144fa..85274f11c67a 100644 --- a/src/gateway/server-methods/environments.ts +++ b/src/gateway/server-methods/environments.ts @@ -7,6 +7,7 @@ import { validateEnvironmentsDestroyParams, validateEnvironmentsListParams, validateEnvironmentsStatusParams, + validateWorkerDesktopObserveParams, } from "../../../packages/gateway-protocol/src/index.js"; import { listNodePairing } from "../../infra/device-pairing-node.js"; import { listDevicePairing, resolveNodePairingState } from "../../infra/device-pairing.js"; @@ -82,6 +83,7 @@ export function summarizeWorkerEnvironment( ...((record.state === "failed" || record.state === "orphaned") && record.error ? { error: record.error } : {}), + ...(record.desktopAvailable ? { desktop: true } : {}), }, }; } @@ -249,4 +251,35 @@ export const environmentsHandlers: GatewayRequestHandlers = { "worker environment destruction failed", ); }, + "worker.desktop.observe": async ({ params, respond, context }) => { + if (!validateWorkerDesktopObserveParams(params)) { + return rejectInvalid(respond, "worker.desktop.observe", validateWorkerDesktopObserveParams); + } + const service = context.workerEnvironmentService; + if (!service) { + respond(false, undefined, errorShape(ErrorCodes.INVALID_REQUEST, "unknown environmentId")); + return; + } + try { + respond( + true, + await service.observeDesktop({ + environmentId: params.environmentId, + control: params.control ?? false, + }), + undefined, + ); + } catch (error) { + const code = error && typeof error === "object" && "code" in error ? error.code : undefined; + const invalid = code === "environment_not_found" || code === "invalid_state"; + respond( + false, + undefined, + errorShape( + invalid ? ErrorCodes.INVALID_REQUEST : ErrorCodes.UNAVAILABLE, + invalid && error instanceof Error ? error.message : "worker desktop observe unavailable", + ), + ); + } + }, }; diff --git a/src/gateway/server-runtime-state-prepare.ts b/src/gateway/server-runtime-state-prepare.ts index b0f49e0015c2..ae2b3df917a4 100644 --- a/src/gateway/server-runtime-state-prepare.ts +++ b/src/gateway/server-runtime-state-prepare.ts @@ -127,7 +127,8 @@ export async function prepareGatewayRuntimeState(params: { }); }) : {}; - const { workerEnvironmentService, workerLiveEvents } = workerEnvironmentRuntime; + const { workerEnvironmentService, workerLiveEvents, workerTunnelManager } = + workerEnvironmentRuntime; // Assigned once approval managers exist; placement dispatch must not run before then. const workerDispatchAuthority = { revoke: (_params: { sessionId: string; sessionKeys: readonly string[] }): void => { @@ -152,6 +153,8 @@ export async function prepareGatewayRuntimeState(params: { const workerPlacementDispatchAvailable = hasConfiguredWorkerProfiles ? workerPlacementControlAvailable : undefined; + const workerDesktopObserveAvailable = + Boolean(workerEnvironmentService) && gatewayPluginConfigAtStart.cloudWorkers?.desktop === true; const channelLogs = Object.fromEntries( listGatewayStartupChannelPlugins().map((plugin) => [plugin.id, logChannels.child(plugin.id)]), ) as Record>; @@ -172,7 +175,8 @@ export async function prepareGatewayRuntimeState(params: { uniqueStrings([...nextBaseGatewayMethods, ...listStartupChannelGatewayMethods()]).filter( (method) => (workerPlacementDispatchAvailable || method !== "sessions.dispatch") && - (workerPlacementControlAvailable || method !== "sessions.reclaim"), + (workerPlacementControlAvailable || method !== "sessions.reclaim") && + (workerDesktopObserveAvailable || method !== "worker.desktop.observe"), ); const runtimeConfig = await startupTrace.measure("runtime.config", async () => { const { resolveGatewayRuntimeConfig } = await import("./server-runtime-config.js"); @@ -407,6 +411,7 @@ export async function prepareGatewayRuntimeState(params: { handleWatchNodeRequest: async (req, res) => (await watchNodeRequestHandler.current?.(req, res)) ?? false, workerIngressEnabled: Boolean(workerEnvironmentService), + workerDesktopTunnels: workerTunnelManager?.desktop, }), ); @@ -420,6 +425,7 @@ export async function prepareGatewayRuntimeState(params: { workerPlacementRuntime, workerPlacementControlAvailable, workerPlacementDispatchAvailable, + workerDesktopObserveAvailable, channelLogs, channelRuntimeEnvs, listStartupChannelGatewayMethods, diff --git a/src/gateway/server-runtime-state.ts b/src/gateway/server-runtime-state.ts index 356ef065f8c6..4bae07655b5a 100644 --- a/src/gateway/server-runtime-state.ts +++ b/src/gateway/server-runtime-state.ts @@ -60,6 +60,7 @@ import type { ReadinessChecker } from "./server/readiness.js"; import type { GatewayTlsRuntime } from "./server/tls.js"; import type { GatewayWsClient } from "./server/ws-types.js"; import { canReceiveSessionEvent } from "./session-sharing.js"; +import type { WorkerDesktopTunnels } from "./worker-environments/desktop-tunnel.js"; type GatewayPluginRequestHandler = ( req: IncomingMessage, @@ -133,6 +134,7 @@ export async function createGatewayRuntimeState(params: { isTerminalEnabled: () => boolean; handleWatchNodeRequest?: (req: IncomingMessage, res: ServerResponse) => Promise; workerIngressEnabled?: boolean; + workerDesktopTunnels?: WorkerDesktopTunnels; }): Promise<{ httpServer: HttpServer; httpServers: HttpServer[]; @@ -340,6 +342,7 @@ export async function createGatewayRuntimeState(params: { getResolvedAuth: params.getResolvedAuth, rateLimiter: params.rateLimiter, log: params.log, + workerDesktopTunnels: params.workerDesktopTunnels, }); gatewayHttpServers.push(httpServer); httpServers.push(httpServer); diff --git a/src/gateway/server-worker-environment-startup.ts b/src/gateway/server-worker-environment-startup.ts index 1d2510fb6526..b7be4f2be45a 100644 --- a/src/gateway/server-worker-environment-startup.ts +++ b/src/gateway/server-worker-environment-startup.ts @@ -10,6 +10,7 @@ import type { WorkerBundleProducer, WorkerNpmArtifact } from "./worker-environme import type { WorkerLiveEventReceiver } from "./worker-environments/live-events.js"; import type { WorkerSessionPlacementStore } from "./worker-environments/placement-store.js"; import type { WorkerEnvironmentService } from "./worker-environments/service.js"; +import type { WorkerTunnelManager } from "./worker-environments/tunnel.js"; type WorkerEnvironmentStore = ReturnType< typeof import("./worker-environments/store.js").createWorkerEnvironmentStore @@ -32,6 +33,7 @@ export type GatewayWorkerEnvironmentStartupState = { export type GatewayWorkerEnvironmentRuntime = { workerEnvironmentService?: WorkerEnvironmentService; workerLiveEvents?: WorkerLiveEventReceiver; + workerTunnelManager?: WorkerTunnelManager; }; const loadWorkerEnvironmentRuntimeModule = createLazyRuntimeModule( @@ -134,13 +136,14 @@ export async function createGatewayWorkerEnvironmentRuntime(params: { startupBindings.map((binding) => [binding.environmentId, binding.runEpoch] as const), ), }); + const workerTunnelManager = createWorkerTunnelManager(); const workerEnvironmentService = createWorkerEnvironmentService({ store: params.startup.store, getConfig: getRuntimeConfig, // Plugin reload replaces the registry object; resolve against the live binding. resolveProvider: (providerId) => resolveWorkerProvider(params.getPluginRegistry(), providerId), prepareInstallation, - tunnelManager: createWorkerTunnelManager(), + tunnelManager: workerTunnelManager, resolveWorkerGateway: params.resolveWorkerGateway, applyTranscriptCommit: createWorkerTranscriptCommitter({ getConfig: getRuntimeConfig, @@ -187,5 +190,5 @@ export async function createGatewayWorkerEnvironmentRuntime(params: { }, logger: params.log.child("worker-environments"), }); - return { workerEnvironmentService, workerLiveEvents }; + return { workerEnvironmentService, workerLiveEvents, workerTunnelManager }; } diff --git a/src/gateway/server.plugin-node-capability-auth.test.ts b/src/gateway/server.plugin-node-capability-auth.test.ts index 8a7615c342ed..e218716a534a 100644 --- a/src/gateway/server.plugin-node-capability-auth.test.ts +++ b/src/gateway/server.plugin-node-capability-auth.test.ts @@ -1,10 +1,17 @@ // Plugin node capability auth tests cover scoped canvas/A2UI HTTP and WebSocket // routes, preauth budgets, capability paths, and unauthorized upgrade handling. +import fs from "node:fs/promises"; import { request, type IncomingMessage, type ServerResponse } from "node:http"; -import { connect, type Socket } from "node:net"; +import { connect, createServer, type Socket } from "node:net"; +import os from "node:os"; +import { join as joinPath } from "node:path"; import type { Duplex } from "node:stream"; -import { describe, expect, test } from "vitest"; +import { describe, expect, test, vi } from "vitest"; import { WebSocket, WebSocketServer } from "ws"; +import { + markGatewayRestartDraining, + resetGatewayWorkAdmission, +} from "../process/gateway-work-admission.js"; import { withTimeout } from "../utils/with-timeout.js"; import { createAuthRateLimiter } from "./auth-rate-limit.js"; import type { ResolvedGatewayAuth } from "./auth.js"; @@ -14,6 +21,10 @@ import { attachGatewayUpgradeHandler, createGatewayHttpServer } from "./server-h import { createPreauthConnectionBudget } from "./server/preauth-connection-budget.js"; import type { GatewayWsClient } from "./server/ws-types.js"; import { withTempConfig } from "./test-temp-config.js"; +import { + mintWorkerDesktopObserverToken, + WORKER_DESKTOP_OBSERVE_PATH, +} from "./worker-environments/desktop-observe.js"; const WS_REJECT_TIMEOUT_MS = 2_000; const WS_CONNECT_TIMEOUT_MS = 5_000; @@ -343,6 +354,10 @@ async function withCanvasGatewayHarness(params: { listenHost?: string; rateLimiter?: ReturnType; handleHttpRequest: CanvasHostHandler["handleHttpRequest"]; + resolvePluginNodeCapabilityRoute?: Parameters< + typeof attachGatewayUpgradeHandler + >[0]["resolvePluginNodeCapabilityRoute"]; + workerDesktopTunnels?: Parameters[0]["workerDesktopTunnels"]; run: (ctx: { listener: Awaited>; clients: Set; @@ -388,7 +403,8 @@ async function withCanvasGatewayHarness(params: { } return canvasHandler.handleHttpRequest(req, res); }, - resolvePluginNodeCapabilityRoute: () => ({ surface: "canvas" }), + resolvePluginNodeCapabilityRoute: + params.resolvePluginNodeCapabilityRoute ?? (() => ({ surface: "canvas" })), resolvedAuth: params.resolvedAuth, getResolvedAuth: params.getResolvedAuth, rateLimiter: params.rateLimiter, @@ -403,12 +419,14 @@ async function withCanvasGatewayHarness(params: { wss, handlePluginUpgrade: async (req, socket, head) => canvasHandler.handleUpgrade(req, socket, head), - resolvePluginNodeCapabilityRoute: () => ({ surface: "canvas" }), + resolvePluginNodeCapabilityRoute: + params.resolvePluginNodeCapabilityRoute ?? (() => ({ surface: "canvas" })), clients, preauthConnectionBudget: createPreauthConnectionBudget(8), resolvedAuth: params.resolvedAuth, getResolvedAuth: params.getResolvedAuth, rateLimiter: params.rateLimiter, + workerDesktopTunnels: params.workerDesktopTunnels, }); const listener = await listen(httpServer, params.listenHost); @@ -421,12 +439,20 @@ async function withCanvasGatewayHarness(params: { for (const ws of wss.clients) { ws.terminate(); } - await new Promise((resolve) => { - canvasWss.close(() => resolve()); - }); - await new Promise((resolve) => { - wss.close(() => resolve()); - }); + await withTimeout( + new Promise((resolve) => { + canvasWss.close(() => resolve()); + }), + SERVER_CLOSE_TIMEOUT_MS, + { message: "canvas websocket server close timed out" }, + ); + await withTimeout( + new Promise((resolve) => { + wss.close(() => resolve()); + }), + SERVER_CLOSE_TIMEOUT_MS, + { message: "gateway websocket server close timed out" }, + ); await listener.close(); params.rateLimiter?.dispose(); } @@ -761,4 +787,93 @@ describe("gateway plugin node capability auth", () => { }, }); }, 60_000); + + test("routes one-shot worker desktop tokens through the real gateway upgrade path", async () => { + const root = await fs.mkdtemp(joinPath(await fs.realpath(os.tmpdir()), "ocwd-")); + const localSocketPath = joinPath(root, "desktop.sock"); + const rfbBytes = Buffer.from("RFB 003.008\n"); + const desktopSockets = new Set(); + const desktopServer = createServer((socket) => { + desktopSockets.add(socket); + socket.once("close", () => desktopSockets.delete(socket)); + socket.write(rfbBytes); + }); + await withTimeout( + new Promise((resolve, reject) => { + desktopServer.once("error", reject); + desktopServer.listen(localSocketPath, () => { + desktopServer.off("error", reject); + resolve(); + }); + }), + 5_000, + { message: "desktop unix server listen timed out" }, + ); + const release = vi.fn(); + const workerDesktopTunnels = { + attachObserver: () => ({ release }), + } as unknown as NonNullable< + Parameters[0]["workerDesktopTunnels"] + >; + try { + await withCanvasGatewayHarness({ + resolvedAuth: tokenResolvedAuth, + handleHttpRequest: async () => false, + resolvePluginNodeCapabilityRoute: () => undefined, + workerDesktopTunnels, + run: async ({ listener }) => { + const minted = mintWorkerDesktopObserverToken({ + environmentId: "worker:boundary", + ownerEpoch: 4, + control: false, + localSocketPath, + }); + const url = `ws://127.0.0.1:${listener.port}${WORKER_DESKTOP_OBSERVE_PATH}?token=${minted.token}`; + const ws = new WebSocket(url); + const received = new Promise((resolve, reject) => { + ws.once("message", (data) => resolve(Buffer.from(data as Buffer))); + ws.once("error", reject); + }); + await expect( + withTimeout(received, 5_000, { message: "desktop RFB bytes timed out" }), + ).resolves.toEqual(rfbBytes); + ws.terminate(); + await vi.waitFor(() => expect(release).toHaveBeenCalledOnce()); + await expectWsRejected(url, {}, 401); + + // A draining Gateway must refuse new desktop observers like every other + // core upgrade; otherwise restart/suspension leaks long-lived sockets. + const draining = mintWorkerDesktopObserverToken({ + environmentId: "worker:boundary", + ownerEpoch: 4, + control: false, + localSocketPath, + }); + markGatewayRestartDraining(); + try { + await expectWsRejected( + `ws://127.0.0.1:${listener.port}${WORKER_DESKTOP_OBSERVE_PATH}?token=${draining.token}`, + {}, + 503, + ); + } finally { + resetGatewayWorkAdmission(); + } + }, + }); + expect(release).toHaveBeenCalledOnce(); + } finally { + for (const socket of desktopSockets) { + socket.destroy(); + } + await withTimeout( + new Promise((resolve) => { + desktopServer.close(() => resolve()); + }), + 5_000, + { message: "desktop unix server close timed out" }, + ); + await fs.rm(root, { recursive: true, force: true }); + } + }, 60_000); }); diff --git a/src/gateway/server.worker-desktop-advertisement.test.ts b/src/gateway/server.worker-desktop-advertisement.test.ts new file mode 100644 index 000000000000..b4554bed35e2 --- /dev/null +++ b/src/gateway/server.worker-desktop-advertisement.test.ts @@ -0,0 +1,37 @@ +// Gateway startup advertisement tests for the cloud-worker Desktop Labs gate. +import { describe, expect, it } from "vitest"; +import { writeConfigFile } from "../config/config.js"; +import { connectOk, installGatewayTestHooks, startServerWithClient } from "./test-helpers.js"; + +installGatewayTestHooks(); + +describe("cloud worker desktop method advertisement", () => { + it.each([ + { desktop: undefined, advertised: false }, + { desktop: true, advertised: true }, + ])("advertises worker.desktop.observe only when the Labs gate is $desktop", async (testCase) => { + process.env.OPENCLAW_TEST_MINIMAL_GATEWAY = "0"; + await writeConfigFile({ + cloudWorkers: { + ...(testCase.desktop === undefined ? {} : { desktop: testCase.desktop }), + profiles: { + development: { + provider: "test-worker-provider", + settings: {}, + }, + }, + }, + }); + const { server, ws } = await startServerWithClient(undefined, { auth: { mode: "none" } }); + try { + const hello = await connectOk(ws); + const methods = (hello as { features?: { methods?: string[] } }).features?.methods ?? []; + + expect(methods).toContain("sessions.dispatch"); + expect(methods.includes("worker.desktop.observe")).toBe(testCase.advertised); + } finally { + ws.close(); + await server.close(); + } + }); +}); diff --git a/src/gateway/worker-environments/desktop-observe.test.ts b/src/gateway/worker-environments/desktop-observe.test.ts new file mode 100644 index 000000000000..4ae72dcc28e2 --- /dev/null +++ b/src/gateway/worker-environments/desktop-observe.test.ts @@ -0,0 +1,221 @@ +import fs from "node:fs/promises"; +import http from "node:http"; +import net from "node:net"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { WebSocket } from "ws"; +import { + handleWorkerDesktopUpgrade, + mintWorkerDesktopObserverToken, + WORKER_DESKTOP_OBSERVE_PATH, +} from "./desktop-observe.js"; + +const cleanup: Array<() => Promise> = []; + +afterEach(async () => { + vi.restoreAllMocks(); + await Promise.all(cleanup.splice(0).map((run) => run())); +}); + +describe("worker desktop observer tokens", () => { + it("mints opaque tokens that expire after 60 seconds", () => { + const minted = mintWorkerDesktopObserverToken({ + environmentId: "worker:one", + ownerEpoch: 3, + control: true, + localSocketPath: "/tmp/desktop.sock", + nowMs: 1_000, + }); + expect(minted.token).toMatch(/^[a-f0-9]{48}$/u); + expect(minted.expiresAtMs).toBe(61_000); + }); +}); + +async function createProxyHarness( + params: { control?: boolean; getBufferedAmount?: () => number } = {}, +) { + const root = await fs.mkdtemp(path.join(await fs.realpath(os.tmpdir()), "desktop-observe-")); + const localSocketPath = path.join(root, "desktop.sock"); + let desktopPeer: net.Socket | undefined; + const peerConnected = new Promise((resolve) => { + const server = net.createServer((socket) => { + desktopPeer = socket; + resolve(socket); + }); + server.listen(localSocketPath); + cleanup.push( + async () => + await new Promise((resolveClose) => { + server.close(() => resolveClose()); + }), + ); + }); + const release = vi.fn(); + const closeObserver = vi.fn(); + const httpServer = http.createServer(); + httpServer.on("upgrade", (req, socket, head) => { + handleWorkerDesktopUpgrade(req, socket, head, { + tunnels: { + attachObserver: (_environmentId, observer) => { + closeObserver.mockImplementation((code: number, reason: string) => { + observer.close(code, reason); + }); + return { release }; + }, + }, + ...(params.getBufferedAmount ? { getBufferedAmount: () => params.getBufferedAmount!() } : {}), + }); + }); + await new Promise((resolve) => { + httpServer.listen(0, "127.0.0.1", resolve); + }); + const address = httpServer.address(); + if (!address || typeof address === "string") { + throw new Error("expected TCP test server address"); + } + cleanup.push(async () => { + desktopPeer?.destroy(); + await new Promise((resolveClose) => { + httpServer.close(() => resolveClose()); + }); + await fs.rm(root, { recursive: true, force: true }); + }); + const minted = mintWorkerDesktopObserverToken({ + environmentId: "worker:pump", + ownerEpoch: 2, + control: params.control ?? false, + localSocketPath, + }); + const ws = new WebSocket( + `ws://127.0.0.1:${address.port}${WORKER_DESKTOP_OBSERVE_PATH}?token=${minted.token}`, + ); + cleanup.push(async () => ws.terminate()); + await new Promise((resolve, reject) => { + ws.once("open", resolve); + ws.once("error", reject); + }); + return { closeObserver, desktopPeer: await peerConnected, observerUrl: ws.url, release, ws }; +} + +function readSocketBytes(socket: net.Socket, byteLength: number): Promise { + return new Promise((resolve) => { + const chunks: Buffer[] = []; + let received = 0; + const onData = (chunk: Buffer) => { + chunks.push(chunk); + received += chunk.length; + if (received >= byteLength) { + socket.off("data", onData); + resolve(Buffer.concat(chunks)); + } + }; + socket.on("data", onData); + }); +} + +async function expectUnauthorizedObserver(url: string): Promise { + const ws = new WebSocket(url); + cleanup.push(async () => ws.terminate()); + await new Promise((resolve, reject) => { + ws.once("open", () => reject(new Error("observer token was unexpectedly accepted"))); + ws.once("unexpected-response", (_request, response) => { + expect(response.statusCode).toBe(401); + response.resume(); + resolve(); + }); + ws.once("error", () => undefined); + }); +} + +describe("worker desktop observer proxy", () => { + it("rejects consumed, expired, and unknown tokens", async () => { + const harness = await createProxyHarness(); + await expectUnauthorizedObserver(harness.observerUrl); + + const expired = mintWorkerDesktopObserverToken({ + environmentId: "worker:expired", + ownerEpoch: 1, + control: false, + localSocketPath: "/tmp/expired.sock", + nowMs: 0, + }); + const observerUrl = new URL(harness.observerUrl); + observerUrl.searchParams.set("token", expired.token); + await expectUnauthorizedObserver(observerUrl.toString()); + observerUrl.searchParams.set("token", "0".repeat(48)); + await expectUnauthorizedObserver(observerUrl.toString()); + }); + + it("drops view-only input while forwarding framebuffer requests", async () => { + const harness = await createProxyHarness(); + const fromDesktop = new Promise((resolve) => { + harness.ws.once("message", (data) => resolve(Buffer.from(data as Buffer))); + }); + harness.desktopPeer.write(Buffer.from("RFB 003.008\n")); + await expect(fromDesktop).resolves.toEqual(Buffer.from("RFB 003.008\n")); + + const handshake = Buffer.concat([Buffer.from("RFB 003.008\n", "ascii"), Buffer.from([1, 1])]); + const keyEvent = Buffer.from([4, 1, 0, 0, 0, 0, 0, 65]); + const framebufferRequest = Buffer.from([3, 1, 0, 0, 0, 0, 0, 64, 0, 64]); + const fromWebSocket = readSocketBytes( + harness.desktopPeer, + handshake.length + framebufferRequest.length, + ); + harness.ws.send(Buffer.concat([handshake, keyEvent, framebufferRequest])); + await expect(fromWebSocket).resolves.toEqual(Buffer.concat([handshake, framebufferRequest])); + + const closed = new Promise((resolve) => { + harness.ws.once("close", () => resolve()); + }); + harness.desktopPeer.destroy(); + await closed; + expect(harness.release).toHaveBeenCalledOnce(); + }); + + it("keeps controlling observers on the plain pass-through path", async () => { + const harness = await createProxyHarness({ control: true }); + const bytes = Buffer.concat([Buffer.from("RFB 003.008\n", "ascii"), Buffer.from([1, 0])]); + const fromWebSocket = readSocketBytes(harness.desktopPeer, bytes.length); + harness.ws.send(bytes); + await expect(fromWebSocket).resolves.toEqual(bytes); + }); + + it("closes malformed view-only streams with a policy violation", async () => { + const harness = await createProxyHarness(); + const closed = new Promise<{ code: number; reason: string }>((resolve) => { + harness.ws.once("close", (code, reason) => resolve({ code, reason: reason.toString() })); + }); + harness.ws.send( + Buffer.concat([Buffer.from("RFB 003.008\n", "ascii"), Buffer.from([1, 1, 255])]), + ); + await expect(closed).resolves.toEqual({ + code: 1008, + reason: "invalid view-only RFB stream", + }); + expect(harness.release).toHaveBeenCalledOnce(); + }); + + it("propagates websocket close to the unix socket", async () => { + const harness = await createProxyHarness(); + const closed = new Promise((resolve) => { + harness.desktopPeer.once("close", resolve); + }); + harness.ws.close(); + await closed; + expect(harness.release).toHaveBeenCalledOnce(); + }); + + it("pauses and resumes unix-socket reads around websocket backpressure", async () => { + let bufferedAmount = 5 * 1024 * 1024; + const pause = vi.spyOn(net.Socket.prototype, "pause"); + const resume = vi.spyOn(net.Socket.prototype, "resume"); + const harness = await createProxyHarness({ getBufferedAmount: () => bufferedAmount }); + pause.mockClear(); + resume.mockClear(); + harness.desktopPeer.write(Buffer.from("RFB")); + await vi.waitFor(() => expect(pause).toHaveBeenCalled()); + bufferedAmount = 0; + await vi.waitFor(() => expect(resume).toHaveBeenCalled()); + }); +}); diff --git a/src/gateway/worker-environments/desktop-observe.ts b/src/gateway/worker-environments/desktop-observe.ts new file mode 100644 index 000000000000..75640c27eaf3 --- /dev/null +++ b/src/gateway/worker-environments/desktop-observe.ts @@ -0,0 +1,181 @@ +import crypto from "node:crypto"; +import type { IncomingMessage } from "node:http"; +import net from "node:net"; +import type { Duplex } from "node:stream"; +import { WebSocket, WebSocketServer, type RawData } from "ws"; +import type { WorkerDesktopTunnels } from "./desktop-tunnel.js"; +import { createRfbClientMessageFilter } from "./rfb-view-only-filter.js"; + +export const WORKER_DESKTOP_OBSERVE_PATH = "/worker-desktop/observe"; +const TOKEN_TTL_MS = 60_000; +const TOKEN_PATTERN = /^[a-f0-9]{48}$/u; +const MAX_PAYLOAD_BYTES = 1024 * 1024; +const PAUSE_BUFFERED_BYTES = 4 * 1024 * 1024; +const RESUME_CHECK_MS = 25; + +type WorkerDesktopObserverTokenEntry = { + environmentId: string; + ownerEpoch: number; + control: boolean; + localSocketPath: string; + expiresAt: number; +}; + +const observerTokens = new Map(); +const desktopObserverWss = new WebSocketServer({ noServer: true, maxPayload: MAX_PAYLOAD_BYTES }); + +function pruneWorkerDesktopObserverTokens(nowMs: number): void { + for (const [token, entry] of observerTokens) { + if (entry.expiresAt <= nowMs) { + observerTokens.delete(token); + } + } +} + +export function mintWorkerDesktopObserverToken(params: { + environmentId: string; + ownerEpoch: number; + control: boolean; + localSocketPath: string; + nowMs?: number; +}): { token: string; expiresAtMs: number } { + const nowMs = params.nowMs ?? Date.now(); + pruneWorkerDesktopObserverTokens(nowMs); + const token = crypto.randomBytes(24).toString("hex"); + const expiresAtMs = nowMs + TOKEN_TTL_MS; + observerTokens.set(token, { + environmentId: params.environmentId, + ownerEpoch: params.ownerEpoch, + control: params.control, + localSocketPath: params.localSocketPath, + expiresAt: expiresAtMs, + }); + return { token, expiresAtMs }; +} + +function consumeWorkerDesktopObserverToken( + token: string, + nowMs = Date.now(), +): WorkerDesktopObserverTokenEntry | undefined { + pruneWorkerDesktopObserverTokens(nowMs); + const normalized = token.trim(); + if (!TOKEN_PATTERN.test(normalized)) { + return undefined; + } + const entry = observerTokens.get(normalized); + if (!entry) { + return undefined; + } + observerTokens.delete(normalized); + return entry.expiresAt > nowMs ? entry : undefined; +} + +function writeUnauthorized(socket: Duplex): void { + socket.write("HTTP/1.1 401 Unauthorized\r\nConnection: close\r\n\r\n"); + socket.destroy(); +} + +function rawDataBuffer(data: RawData): Buffer { + if (Buffer.isBuffer(data)) { + return data; + } + if (Array.isArray(data)) { + return Buffer.concat(data); + } + return Buffer.from(data); +} + +/** Upgrades one authenticated observer token into a raw bidirectional RFB stream. */ +export function handleWorkerDesktopUpgrade( + req: IncomingMessage, + socket: Duplex, + head: Buffer, + deps: { + tunnels: Pick; + getBufferedAmount?: (ws: WebSocket) => number; + }, +): boolean { + const resource = new URL(req.url ?? "/", "http://127.0.0.1"); + if (resource.pathname !== WORKER_DESKTOP_OBSERVE_PATH) { + return false; + } + const token = resource.searchParams.get("token") ?? ""; + const entry = consumeWorkerDesktopObserverToken(token); + if (!entry) { + writeUnauthorized(socket); + return true; + } + desktopObserverWss.handleUpgrade(req, socket, head, (ws) => { + // View-only is enforced here at the RFB message boundary; the UI setting is only UX. + const observer = deps.tunnels.attachObserver(entry.environmentId, { + control: entry.control, + ownerEpoch: entry.ownerEpoch, + close: (code, reason) => ws.close(code, reason), + }); + if (!observer) { + ws.close(1013, "desktop observer limit"); + return; + } + const desktopSocket = net.connect(entry.localSocketPath); + const clientMessageFilter = entry.control ? undefined : createRfbClientMessageFilter(); + let closed = false; + let resumeTimer: ReturnType | undefined; + + const closeBoth = (code: number, reason: string) => { + if (closed) { + return; + } + closed = true; + clearInterval(resumeTimer); + resumeTimer = undefined; + observer.release(); + desktopSocket.destroy(); + if (ws.readyState === WebSocket.OPEN || ws.readyState === WebSocket.CONNECTING) { + ws.close(code, reason); + } + }; + + ws.on("message", (data, isBinary) => { + if (!isBinary || closed) { + return; + } + const chunk = rawDataBuffer(data); + if (!clientMessageFilter) { + desktopSocket.write(chunk); + return; + } + const result = clientMessageFilter.filter(chunk); + if ("error" in result) { + closeBoth(1008, "invalid view-only RFB stream"); + return; + } + if (result.forward.length > 0) { + desktopSocket.write(result.forward); + } + }); + ws.once("close", () => closeBoth(1000, "desktop observer closed")); + ws.once("error", () => closeBoth(1011, "desktop observer failed")); + desktopSocket.on("data", (chunk) => { + if (closed || ws.readyState !== WebSocket.OPEN) { + return; + } + ws.send(chunk, { binary: true }); + const bufferedAmount = () => deps.getBufferedAmount?.(ws) ?? ws.bufferedAmount; + if (bufferedAmount() <= PAUSE_BUFFERED_BYTES || resumeTimer) { + return; + } + desktopSocket.pause(); + resumeTimer = setInterval(() => { + if (bufferedAmount() <= PAUSE_BUFFERED_BYTES) { + clearInterval(resumeTimer); + resumeTimer = undefined; + desktopSocket.resume(); + } + }, RESUME_CHECK_MS); + resumeTimer.unref?.(); + }); + desktopSocket.once("close", () => closeBoth(1000, "desktop stream closed")); + desktopSocket.once("error", () => closeBoth(1011, "desktop stream failed")); + }); + return true; +} diff --git a/src/gateway/worker-environments/desktop-tunnel.test.ts b/src/gateway/worker-environments/desktop-tunnel.test.ts new file mode 100644 index 000000000000..9c34f0859e0a --- /dev/null +++ b/src/gateway/worker-environments/desktop-tunnel.test.ts @@ -0,0 +1,274 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { WorkerDesktopEndpoint, WorkerSshEndpoint } from "../../plugins/types.js"; +import type { CommandOptions, SpawnResult } from "../../process/exec.js"; +import { createWorkerDesktopTunnels } from "./desktop-tunnel.js"; +import type { WorkerSshProcess, WorkerSshRunner } from "./tunnel-ssh-runner.js"; + +const SSH: WorkerSshEndpoint = { + host: "worker.example.test", + port: 2202, + user: "worker", + hostKey: "ssh-ed25519 AAAA", + keyRef: { source: "file", provider: "workers", id: "/identity" }, +}; +const DESKTOP: WorkerDesktopEndpoint = { + protocol: "rfb", + port: 5900, + passwordFilePath: "/var/lib/crabbox/vnc.password", +}; +const resolveIdentity = async () => ({ kind: "path", path: "/keys/worker" }) as const; + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (error: Error) => void; + const promise = new Promise((promiseResolve, promiseReject) => { + resolve = promiseResolve; + reject = promiseReject; + }); + void promise.catch(() => undefined); + return { promise, resolve, reject }; +} + +class FakeProcess implements WorkerSshProcess { + private readonly readyDeferred = deferred(); + private readonly exitDeferred = deferred<{ + code: number | null; + signal: NodeJS.Signals | null; + }>(); + readonly ready = this.readyDeferred.promise; + readonly exited = this.exitDeferred.promise; + stopCount = 0; + private stopPromise?: Promise; + + becomeReady() { + this.readyDeferred.resolve(); + } + + exit() { + this.exitDeferred.resolve({ code: 1, signal: null }); + } + + stop() { + return (this.stopPromise ??= Promise.resolve().then(() => { + this.stopCount += 1; + this.readyDeferred.reject(new Error("stopped")); + this.exitDeferred.resolve({ code: null, signal: "SIGTERM" }); + })); + } +} + +function success(stdout = ""): SpawnResult { + return { + stdout, + stderr: "", + code: 0, + signal: null, + killed: false, + termination: "exit", + }; +} + +function fakeRunner() { + const starts: Array<{ argv: string[]; options: CommandOptions; process: FakeProcess }> = []; + const runs: Array<{ argv: string[]; options: CommandOptions }> = []; + const runner: WorkerSshRunner = { + start(argv, options) { + const process = new FakeProcess(); + starts.push({ argv, options, process }); + return process; + }, + async run(argv, options) { + runs.push({ argv, options }); + return success("vnc-secret\n"); + }, + }; + return { runner, runs, starts }; +} + +function acquire( + manager: ReturnType, + ownerEpoch = 1, + desktop = DESKTOP, +) { + return manager.acquire({ + environmentId: "worker:one", + ownerEpoch, + ssh: SSH, + desktop, + resolveIdentity, + }); +} + +async function waitForStarts(starts: unknown[], count: number) { + await vi.waitFor(() => expect(starts).toHaveLength(count), { interval: 1 }); +} + +afterEach(() => vi.useRealTimers()); + +describe("worker desktop tunnels", () => { + it("creates one pinned local forward per epoch and caches the password", async () => { + const fake = fakeRunner(); + const manager = createWorkerDesktopTunnels({ runner: fake.runner }); + const starting = acquire(manager); + await waitForStarts(fake.starts, 1); + const start = fake.starts[0]!; + expect(start.argv).toContain("ClearAllForwardings=no"); + expect(start.argv).toContain("StreamLocalBindMask=0177"); + expect(start.argv).toContain("ServerAliveInterval=15"); + expect(start.argv).toContain("ServerAliveCountMax=3"); + expect(start.argv[start.argv.indexOf("-L") + 1]).toMatch( + /openclaw-worker-desktop-.+\/desktop\.sock:127\.0\.0\.1:5900$/u, + ); + expect(start.options.input).toContain("OPENCLAW_WORKER_TUNNEL_READY"); + start.process.becomeReady(); + const result = await starting; + expect(result).toMatchObject({ vncPassword: "vnc-secret" }); + expect(fake.runs).toHaveLength(1); + expect(fake.runs[0]?.argv.at(-1)).toContain("/var/lib/crabbox/vnc.password"); + + await expect(acquire(manager)).resolves.toEqual(result); + expect(fake.starts).toHaveLength(1); + expect(fake.runs).toHaveLength(1); + await manager.stopAll(); + }); + + it("fences an older epoch before starting its replacement", async () => { + const fake = fakeRunner(); + const manager = createWorkerDesktopTunnels({ runner: fake.runner }); + const first = acquire(manager, 1, { protocol: "rfb", port: 5900 }); + await waitForStarts(fake.starts, 1); + fake.starts[0]?.process.becomeReady(); + await first; + + const second = acquire(manager, 2, { protocol: "rfb", port: 5901 }); + await waitForStarts(fake.starts, 2); + expect(fake.starts[0]?.process.stopCount).toBe(1); + expect(fake.starts[1]?.argv[fake.starts[1]!.argv.indexOf("-L") + 1]).toContain( + ":127.0.0.1:5901", + ); + fake.starts[1]?.process.becomeReady(); + await second; + await expect(acquire(manager, 1)).rejects.toThrow("owner epoch is stale"); + await manager.stopAll(); + }); + + it("fences stop by owner epoch while allowing matching and unconditional teardown", async () => { + const fake = fakeRunner(); + const manager = createWorkerDesktopTunnels({ runner: fake.runner }); + const second = acquire(manager, 2, { protocol: "rfb", port: 5900 }); + await waitForStarts(fake.starts, 1); + fake.starts[0]?.process.becomeReady(); + await second; + + await manager.stop("worker:one", 1); + expect(fake.starts[0]?.process.stopCount).toBe(0); + await manager.stop("worker:one", 2); + expect(fake.starts[0]?.process.stopCount).toBe(1); + + const third = acquire(manager, 3, { protocol: "rfb", port: 5900 }); + await waitForStarts(fake.starts, 2); + fake.starts[1]?.process.becomeReady(); + await third; + await manager.stop("worker:one"); + expect(fake.starts[1]?.process.stopCount).toBe(1); + }); + + it("enforces controller takeover and the observer cap", async () => { + const fake = fakeRunner(); + const manager = createWorkerDesktopTunnels({ runner: fake.runner }); + const starting = acquire(manager, 1, { protocol: "rfb", port: 5900 }); + await waitForStarts(fake.starts, 1); + fake.starts[0]?.process.becomeReady(); + await starting; + + const firstClose = vi.fn(); + const first = manager.attachObserver("worker:one", { + control: true, + ownerEpoch: 1, + close: firstClose, + }); + const second = manager.attachObserver("worker:one", { + control: true, + ownerEpoch: 1, + close: vi.fn(), + }); + expect(firstClose).toHaveBeenCalledWith(4000, "control-taken"); + first?.release(); + const observers = Array.from({ length: 7 }, () => + manager.attachObserver("worker:one", { control: false, ownerEpoch: 1, close: vi.fn() }), + ); + expect(observers.every(Boolean)).toBe(true); + expect( + manager.attachObserver("worker:one", { control: false, ownerEpoch: 1, close: vi.fn() }), + ).toBeUndefined(); + second?.release(); + observers.forEach((observer) => observer?.release()); + await manager.stopAll(); + }); + + it("lingers after the last observer and closes observers on child exit", async () => { + vi.useFakeTimers(); + const fake = fakeRunner(); + const manager = createWorkerDesktopTunnels({ runner: fake.runner, lingerMs: 50 }); + const starting = acquire(manager, 1, { protocol: "rfb", port: 5900 }); + await vi.waitFor(() => expect(fake.starts).toHaveLength(1)); + fake.starts[0]?.process.becomeReady(); + await starting; + const close = vi.fn(); + const observer = manager.attachObserver("worker:one", { control: false, ownerEpoch: 1, close }); + observer?.release(); + await vi.advanceTimersByTimeAsync(49); + expect(fake.starts[0]?.process.stopCount).toBe(0); + const replacement = manager.attachObserver("worker:one", { + control: false, + ownerEpoch: 1, + close, + }); + await vi.advanceTimersByTimeAsync(50); + expect(fake.starts[0]?.process.stopCount).toBe(0); + fake.starts[0]?.process.exit(); + await vi.waitFor(() => expect(close).toHaveBeenCalledWith(1012, "desktop tunnel closed")); + replacement?.release(); + }); + + it("refuses observer tokens minted against a replaced owner epoch", async () => { + const fake = fakeRunner(); + const manager = createWorkerDesktopTunnels({ runner: fake.runner }); + const first = acquire(manager, 1); + await waitForStarts(fake.starts, 1); + fake.starts[0]?.process.becomeReady(); + await first; + + const second = acquire(manager, 2); + await waitForStarts(fake.starts, 2); + fake.starts[1]?.process.becomeReady(); + await second; + + const controllerClose = vi.fn(); + const controller = manager.attachObserver("worker:one", { + control: true, + ownerEpoch: 2, + close: controllerClose, + }); + expect(controller).toBeDefined(); + + // A stale control token must not reach the replacement entry or evict its controller. + expect( + manager.attachObserver("worker:one", { control: true, ownerEpoch: 1, close: vi.fn() }), + ).toBeUndefined(); + expect(controllerClose).not.toHaveBeenCalled(); + + controller?.release(); + await manager.stopAll(); + }); + + it("rejects Windows gateway hosts before spawning SSH", async () => { + const fake = fakeRunner(); + const manager = createWorkerDesktopTunnels({ runner: fake.runner, platform: "win32" }); + await expect(acquire(manager)).rejects.toMatchObject({ code: "unsupported_platform" }); + await expect(acquire(manager)).rejects.toThrow( + "desktop observe is not supported on Windows gateway hosts", + ); + expect(fake.starts).toEqual([]); + }); +}); diff --git a/src/gateway/worker-environments/desktop-tunnel.ts b/src/gateway/worker-environments/desktop-tunnel.ts new file mode 100644 index 000000000000..10b0ce1b80d5 --- /dev/null +++ b/src/gateway/worker-environments/desktop-tunnel.ts @@ -0,0 +1,321 @@ +import path from "node:path"; +import { registerSecretValueForRedaction } from "../../logging/secret-redaction-registry.js"; +import type { WorkerDesktopEndpoint, WorkerSshEndpoint } from "../../plugins/types.js"; +import { + prepareWorkerSsh, + type PreparedWorkerSsh, + type WorkerSshIdentityResolver, + workerSshCommandOptions, + workerSshOptions, + workerSshRemoteCommand, +} from "./ssh.js"; +import { + type WorkerSshProcess, + type WorkerSshRunner, + workerSshProcessError, + WORKER_TUNNEL_READY_MARKER, +} from "./tunnel-ssh-runner.js"; + +const DEFAULT_LINGER_MS = 60_000; +const PASSWORD_READ_TIMEOUT_MS = 20_000; +const MAX_OBSERVERS = 8; + +const REMOTE_DESKTOP_READY_SCRIPT = String.raw`set -eu +printf '%s\n' '${WORKER_TUNNEL_READY_MARKER}' +trap 'exit 0' HUP INT TERM +while :; do sleep 3600; done +`; + +type WorkerDesktopObserver = { + control: boolean; + /** Epoch the observer token was minted against; a stale token must not reach a newer entry. */ + ownerEpoch: number; + close(code: number, reason: string): void; +}; + +type DesktopAcquireRequest = { + environmentId: string; + ownerEpoch: number; + ssh: WorkerSshEndpoint; + desktop: WorkerDesktopEndpoint; + resolveIdentity: WorkerSshIdentityResolver; +}; + +type DesktopAcquireResult = { localSocketPath: string; vncPassword?: string }; +type ObserverEntry = WorkerDesktopObserver & { released: boolean }; +type DesktopEntry = { + environmentId: string; + ownerEpoch: number; + localSocketPath?: string; + prepared?: PreparedWorkerSsh; + process?: WorkerSshProcess; + initialization?: Promise; + stopPromise?: Promise; + ready: Promise; + resolveReady: (result: DesktopAcquireResult) => void; + rejectReady: (error: Error) => void; + readySettled: boolean; + observers: Set; + controller?: ObserverEntry; + lingerTimer?: ReturnType; + stopped: boolean; +}; + +class WorkerDesktopUnsupportedError extends Error { + readonly code = "unsupported_platform"; + + constructor() { + super("desktop observe is not supported on Windows gateway hosts"); + this.name = "WorkerDesktopUnsupportedError"; + } +} + +function successful(result: Awaited>): boolean { + return result.termination === "exit" && result.code === 0; +} + +/** Owns per-environment local desktop forwards and their connected observer lifetimes. */ +export function createWorkerDesktopTunnels(deps: { + runner: WorkerSshRunner; + now?: () => number; + lingerMs?: number; + platform?: NodeJS.Platform; +}) { + const lingerMs = deps.lingerMs ?? DEFAULT_LINGER_MS; + const platform = deps.platform ?? process.platform; + const entries = new Map(); + const claimedOwnerEpochs = new Map(); + + const isCurrent = (entry: DesktopEntry) => + entries.get(entry.environmentId) === entry && !entry.stopped; + + const closeObserver = (observer: ObserverEntry, code: number, reason: string) => { + try { + observer.close(code, reason); + } catch { + // Observer cleanup remains authoritative when the transport close callback fails. + } + }; + + const stopEntry = (entry: DesktopEntry): Promise => { + if (entry.stopPromise) { + return entry.stopPromise; + } + entry.stopPromise = (async () => { + entry.stopped = true; + if (entries.get(entry.environmentId) === entry) { + entries.delete(entry.environmentId); + } + clearTimeout(entry.lingerTimer); + entry.lingerTimer = undefined; + for (const observer of entry.observers) { + observer.released = true; + closeObserver(observer, 1012, "desktop tunnel closed"); + } + entry.observers.clear(); + entry.controller = undefined; + if (!entry.readySettled) { + entry.readySettled = true; + entry.rejectReady(new Error("Worker desktop tunnel stopped before connecting")); + } + const processBeforeInitialization = entry.process; + await processBeforeInitialization?.stop().catch(() => undefined); + await entry.initialization?.catch(() => undefined); + if (entry.process !== processBeforeInitialization) { + await entry.process?.stop().catch(() => undefined); + } + await entry.prepared?.dispose().catch(() => undefined); + })(); + return entry.stopPromise; + }; + + const startEntry = async (entry: DesktopEntry, request: DesktopAcquireRequest) => { + const prepared = await prepareWorkerSsh({ + ssh: request.ssh, + pinnedHostKey: request.ssh.hostKey, + resolveIdentity: request.resolveIdentity, + temporaryDirectoryPrefix: "openclaw-worker-desktop-", + }); + entry.prepared = prepared; + if (!isCurrent(entry)) { + await prepared.dispose(); + entry.prepared = undefined; + return; + } + const localSocketPath = path.join(path.dirname(prepared.knownHostsPath), "desktop.sock"); + entry.localSocketPath = localSocketPath; + const child = deps.runner.start( + [ + "ssh", + ...workerSshOptions(prepared, { forwarding: "explicit" }), + "-a", + "-x", + "-T", + "-o", + "ServerAliveInterval=15", + "-o", + "ServerAliveCountMax=3", + "-o", + "StreamLocalBindMask=0177", + "-L", + `${localSocketPath}:127.0.0.1:${request.desktop.port}`, + "-p", + String(prepared.port), + "--", + prepared.sshTarget, + workerSshRemoteCommand(["sh", "-s"]), + ], + workerSshCommandOptions({ + input: REMOTE_DESKTOP_READY_SCRIPT, + timeoutMs: Number.MAX_SAFE_INTEGER, + }), + ); + entry.process = child; + void child.exited.then(() => { + if (isCurrent(entry)) { + void stopEntry(entry); + } + }); + await child.ready; + if (!isCurrent(entry)) { + await child.stop(); + return; + } + let vncPassword: string | undefined; + if (request.desktop.passwordFilePath) { + const result = await deps.runner.run( + [ + "ssh", + ...workerSshOptions(prepared, { forwarding: "disabled" }), + "-a", + "-x", + "-T", + "-p", + String(prepared.port), + "--", + prepared.sshTarget, + workerSshRemoteCommand(["cat", request.desktop.passwordFilePath]), + ], + workerSshCommandOptions({ timeoutMs: PASSWORD_READ_TIMEOUT_MS }), + ); + if (!successful(result)) { + throw workerSshProcessError(result.stderr || result.stdout); + } + vncPassword = result.stdout.replace(/(?:\r?\n)+$/u, ""); + if (!vncPassword) { + throw new Error("Worker desktop password file is empty"); + } + registerSecretValueForRedaction(vncPassword); + } + entry.readySettled = true; + entry.resolveReady({ localSocketPath, ...(vncPassword ? { vncPassword } : {}) }); + }; + + async function acquire(request: DesktopAcquireRequest): Promise { + if (platform === "win32") { + throw new WorkerDesktopUnsupportedError(); + } + const claimedEpoch = claimedOwnerEpochs.get(request.environmentId); + if (claimedEpoch !== undefined && request.ownerEpoch < claimedEpoch) { + throw new Error("Worker desktop tunnel owner epoch is stale"); + } + claimedOwnerEpochs.set(request.environmentId, request.ownerEpoch); + const current = entries.get(request.environmentId); + if (current?.ownerEpoch === request.ownerEpoch) { + return await current.ready; + } + let resolveReady!: (result: DesktopAcquireResult) => void; + let rejectReady!: (error: Error) => void; + const ready = new Promise((resolve, reject) => { + resolveReady = resolve; + rejectReady = reject; + }); + void ready.catch(() => undefined); + const entry: DesktopEntry = { + environmentId: request.environmentId, + ownerEpoch: request.ownerEpoch, + ready, + resolveReady, + rejectReady, + readySettled: false, + observers: new Set(), + stopped: false, + }; + entries.set(request.environmentId, entry); + entry.initialization = (async () => { + if (current) { + await stopEntry(current); + } + if (isCurrent(entry)) { + await startEntry(entry, request); + } + })(); + void entry.initialization.catch((error: unknown) => { + if (!entry.readySettled) { + entry.readySettled = true; + entry.rejectReady( + error instanceof Error ? error : new Error("Worker desktop tunnel failed"), + ); + } + void stopEntry(entry); + }); + return await ready; + } + + function attachObserver(environmentId: string, observer: WorkerDesktopObserver) { + const entry = entries.get(environmentId); + if (!entry || !entry.readySettled || entry.stopped || entry.observers.size >= MAX_OBSERVERS) { + return undefined; + } + // A token minted against a replaced entry must not reach this one; otherwise a stale + // control token would evict the current controller of a desktop it never observed. + if (observer.ownerEpoch !== entry.ownerEpoch) { + return undefined; + } + clearTimeout(entry.lingerTimer); + entry.lingerTimer = undefined; + if (observer.control && entry.controller) { + const previous = entry.controller; + previous.released = true; + entry.observers.delete(previous); + entry.controller = undefined; + closeObserver(previous, 4000, "control-taken"); + } + const attached: ObserverEntry = { ...observer, released: false }; + entry.observers.add(attached); + if (attached.control) { + entry.controller = attached; + } + return { + release() { + if (attached.released) { + return; + } + attached.released = true; + entry.observers.delete(attached); + if (entry.controller === attached) { + entry.controller = undefined; + } + if (entry.observers.size === 0 && isCurrent(entry)) { + entry.lingerTimer = setTimeout(() => void stopEntry(entry), lingerMs); + entry.lingerTimer.unref?.(); + } + }, + }; + } + + async function stop(environmentId: string, ownerEpoch?: number): Promise { + const entry = entries.get(environmentId); + if (entry && (ownerEpoch === undefined || ownerEpoch === entry.ownerEpoch)) { + await stopEntry(entry); + } + } + + async function stopAll(): Promise { + await Promise.all([...entries.values()].map(stopEntry)); + } + + return { acquire, attachObserver, stop, stopAll }; +} + +export type WorkerDesktopTunnels = ReturnType; diff --git a/src/gateway/worker-environments/placement-dispatch-test-fixtures.ts b/src/gateway/worker-environments/placement-dispatch-test-fixtures.ts index 0d91cebedccb..e8e73a08d1bd 100644 --- a/src/gateway/worker-environments/placement-dispatch-test-fixtures.ts +++ b/src/gateway/worker-environments/placement-dispatch-test-fixtures.ts @@ -115,6 +115,8 @@ export function createDispatchEnvironmentFixtures(generation = 1) { destroyRequestedAtMs: null, leaseId: "lease-1", sshEndpoint, + desktop: null, + desktopAvailable: false, }; const ready = { ...environmentBase, diff --git a/src/gateway/worker-environments/rfb-view-only-filter.test.ts b/src/gateway/worker-environments/rfb-view-only-filter.test.ts new file mode 100644 index 000000000000..567c5b401690 --- /dev/null +++ b/src/gateway/worker-environments/rfb-view-only-filter.test.ts @@ -0,0 +1,135 @@ +import { describe, expect, it } from "vitest"; +import { createRfbClientMessageFilter } from "./rfb-view-only-filter.js"; + +const VERSION = Buffer.from("RFB 003.008\n", "ascii"); + +function noneHandshake(): Buffer { + return Buffer.concat([VERSION, Buffer.from([1, 1])]); +} + +function vncAuthHandshake(): Buffer { + return Buffer.concat([ + VERSION, + Buffer.from([2]), + Buffer.from(Array.from({ length: 16 }, (_, index) => index)), + Buffer.from([1]), + ]); +} + +function enterMessagePhase() { + const filter = createRfbClientMessageFilter(); + expect(filter.filter(noneHandshake())).toEqual({ forward: noneHandshake() }); + return filter; +} + +describe("RFB view-only client message filter", () => { + it.each([ + ["None", noneHandshake()], + ["VncAuth", vncAuthHandshake()], + ])("forwards a complete %s handshake byte-identically", (_name, handshake) => { + const filter = createRfbClientMessageFilter(); + expect(filter.filter(handshake)).toEqual({ forward: handshake }); + }); + + it.each([ + ["rewrites exclusive", 0, 1], + ["forwards shared", 1, 1], + ])("%s ClientInit when it arrives in its own chunk", (_name, clientInit, expected) => { + const filter = createRfbClientMessageFilter(); + const prefix = Buffer.concat([VERSION, Buffer.from([1])]); + expect(filter.filter(prefix)).toEqual({ forward: prefix }); + expect(filter.filter(Buffer.from([clientInit]))).toEqual({ + forward: Buffer.from([expected]), + }); + }); + + it("rewrites an exclusive ClientInit batched with the following message", () => { + const filter = createRfbClientMessageFilter(); + const prefix = Buffer.concat([VERSION, Buffer.from([1])]); + const framebufferRequest = Buffer.from([3, 1, 0, 0, 0, 0, 0, 64, 0, 64]); + expect(filter.filter(prefix)).toEqual({ forward: prefix }); + expect(filter.filter(Buffer.concat([Buffer.from([0]), framebufferRequest]))).toEqual({ + forward: Buffer.concat([Buffer.from([1]), framebufferRequest]), + }); + }); + + it("fails closed on unsupported security types", () => { + const filter = createRfbClientMessageFilter(); + expect(filter.filter(Buffer.concat([VERSION, Buffer.from([19])]))).toEqual({ + error: "unsupported RFB security type 19", + }); + }); + + it("drops input messages while forwarding display configuration and update requests", () => { + const filter = enterMessagePhase(); + const setPixelFormat = Buffer.alloc(20); + setPixelFormat[0] = 0; + const setEncodings = Buffer.alloc(8); + setEncodings[0] = 2; + setEncodings.writeUInt16BE(1, 2); + setEncodings.writeInt32BE(0, 4); + const framebufferUpdateRequest = Buffer.from([3, 1, 0, 0, 0, 0, 0, 64, 0, 64]); + const keyEvent = Buffer.from([4, 1, 0, 0, 0, 0, 0, 65]); + const pointerEvent = Buffer.from([5, 1, 0, 10, 0, 20]); + const cutText = Buffer.concat([Buffer.from([6, 0, 0, 0, 0, 0, 0, 3]), Buffer.from("abc")]); + + const result = filter.filter( + Buffer.concat([ + keyEvent, + setPixelFormat, + pointerEvent, + setEncodings, + cutText, + framebufferUpdateRequest, + ]), + ); + + expect(result).toEqual({ + forward: Buffer.concat([setPixelFormat, setEncodings, framebufferUpdateRequest]), + }); + }); + + it("reassembles a message split across three chunks", () => { + const filter = enterMessagePhase(); + const completePrefix = Buffer.from([3, 1, 0, 0, 0, 0, 0, 64, 0, 64]); + const message = Buffer.alloc(20); + message[0] = 0; + message[4] = 32; + + expect(filter.filter(Buffer.concat([completePrefix, message.subarray(0, 3)]))).toEqual({ + forward: completePrefix, + }); + expect(filter.filter(message.subarray(3, 11))).toEqual({ forward: Buffer.alloc(0) }); + expect(filter.filter(message.subarray(11))).toEqual({ forward: message }); + }); + + it("uses the SetEncodings count to route a multi-encoding payload", () => { + const filter = enterMessagePhase(); + const message = Buffer.alloc(16); + message[0] = 2; + message.writeUInt16BE(3, 2); + message.writeInt32BE(0, 4); + message.writeInt32BE(16, 8); + message.writeInt32BE(-223, 12); + + expect(filter.filter(message.subarray(0, 7))).toEqual({ forward: Buffer.alloc(0) }); + expect(filter.filter(message.subarray(7))).toEqual({ forward: message }); + }); + + it("fails closed on unknown message types", () => { + const filter = enterMessagePhase(); + expect(filter.filter(Buffer.from([255]))).toEqual({ + error: "unsupported RFB client message type 255", + }); + }); + + it("fails closed before buffering an oversized variable-length message", () => { + const filter = enterMessagePhase(); + const header = Buffer.alloc(8); + header[0] = 6; + header.writeUInt32BE(64 * 1024, 4); + expect(filter.filter(header)).toEqual({ + error: "RFB client message exceeds the 64 KiB buffer limit", + }); + }); +}); diff --git a/src/gateway/worker-environments/rfb-view-only-filter.ts b/src/gateway/worker-environments/rfb-view-only-filter.ts new file mode 100644 index 000000000000..556b058ae90a --- /dev/null +++ b/src/gateway/worker-environments/rfb-view-only-filter.ts @@ -0,0 +1,121 @@ +const RFB_3_8_VERSION = Buffer.from("RFB 003.008\n", "ascii"); +const MAX_PENDING_BYTES = 64 * 1024; + +type RfbClientPhase = "version" | "security" | "authResponse" | "clientInit" | "messages"; +const FIXED_PHASE_LENGTHS: Record, number> = { + version: RFB_3_8_VERSION.length, + security: 1, + authResponse: 16, + clientInit: 1, +}; + +type RfbClientMessageFilterResult = + | { forward: Buffer; error?: never } + | { forward?: never; error: string }; + +/** Filters one view-only RFB client byte stream without trusting WebSocket frame boundaries. */ +export function createRfbClientMessageFilter() { + let phase: RfbClientPhase = "version"; + let pending = Buffer.alloc(0); + let failure: string | undefined; + + const fail = (error: string): RfbClientMessageFilterResult => { + failure = error; + pending = Buffer.alloc(0); + return { error }; + }; + + const pendingTargetLength = (): number | string => { + if (phase !== "messages") { + return FIXED_PHASE_LENGTHS[phase]; + } + if (pending.length === 0) { + return 1; + } + switch (pending[0]) { + case 0: + return 20; + case 2: + return pending.length < 4 ? 4 : 4 + pending.readUInt16BE(2) * 4; + case 3: + return 10; + case 4: + return 8; + case 5: + return 6; + case 6: + return pending.length < 8 ? 8 : 8 + pending.readUInt32BE(4); + default: + return `unsupported RFB client message type ${pending[0]}`; + } + }; + + const routePending = (forwarded: Buffer[]): string | undefined => { + if (phase === "version") { + if (!pending.equals(RFB_3_8_VERSION)) { + return "unsupported RFB protocol version"; + } + forwarded.push(pending); + phase = "security"; + } else if (phase === "security") { + const securityType = pending[0]; + forwarded.push(pending); + if (securityType === 1) { + phase = "clientInit"; + } else if (securityType === 2) { + phase = "authResponse"; + } else { + return `unsupported RFB security type ${securityType}`; + } + } else if (phase === "authResponse") { + forwarded.push(pending); + phase = "clientInit"; + } else if (phase === "clientInit") { + // A passive viewer must stay shared; exclusive ClientInit would disconnect the controller. + pending[0] = 1; + forwarded.push(pending); + phase = "messages"; + } else if (pending[0] === 0 || pending[0] === 2 || pending[0] === 3) { + forwarded.push(pending); + } + pending = Buffer.alloc(0); + return undefined; + }; + + return { + filter(chunk: Buffer): RfbClientMessageFilterResult { + if (failure) { + return { error: failure }; + } + const forwarded: Buffer[] = []; + let offset = 0; + while (offset < chunk.length) { + const target = pendingTargetLength(); + if (typeof target === "string") { + return fail(target); + } + if (target > MAX_PENDING_BYTES) { + return fail("RFB client message exceeds the 64 KiB buffer limit"); + } + const take = Math.min(target - pending.length, chunk.length - offset); + pending = Buffer.concat([pending, chunk.subarray(offset, offset + take)]); + offset += take; + const completedTarget = pendingTargetLength(); + if (typeof completedTarget === "string") { + return fail(completedTarget); + } + if (completedTarget > MAX_PENDING_BYTES) { + return fail("RFB client message exceeds the 64 KiB buffer limit"); + } + if (pending.length < completedTarget) { + continue; + } + const error = routePending(forwarded); + if (error) { + return fail(error); + } + } + return { forward: Buffer.concat(forwarded) }; + }, + }; +} diff --git a/src/gateway/worker-environments/service-contract.ts b/src/gateway/worker-environments/service-contract.ts index f14bc7a5bb7c..d271adafc4bc 100644 --- a/src/gateway/worker-environments/service-contract.ts +++ b/src/gateway/worker-environments/service-contract.ts @@ -16,10 +16,19 @@ export type WorkerEnvironmentServiceRecord = { createdAtMs: number; idleSinceAtMs: number | null; attachedSessionIds: readonly string[]; + desktopAvailable: boolean; tunnelStatus: WorkerTunnelStatus; error?: string; }; +export type WorkerDesktopObserveResult = { + transport: "rfb"; + wsPath: string; + expiresAtMs: number; + control: boolean; + vncPassword?: string; +}; + /** Request-facing lifecycle methods, kept separate from persistence and provider internals. */ export type WorkerEnvironmentServiceContract = { list(): WorkerEnvironmentServiceRecord[]; @@ -27,6 +36,10 @@ export type WorkerEnvironmentServiceContract = { create(profileId: string, idempotencyKey: string): Promise; destroy(environmentId: string): Promise; destroyUnattached(environmentId: string): Promise; + observeDesktop(request: { + environmentId: string; + control: boolean; + }): Promise; startTunnel(request: WorkerTunnelRequest): Promise; stopTunnel(environmentId: string, ownerEpoch?: number): Promise; }; diff --git a/src/gateway/worker-environments/service-validation.ts b/src/gateway/worker-environments/service-validation.ts index 64460cdfab18..937ea7f46fa2 100644 --- a/src/gateway/worker-environments/service-validation.ts +++ b/src/gateway/worker-environments/service-validation.ts @@ -2,8 +2,13 @@ import { isRecord } from "@openclaw/normalization-core/record-coerce"; import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; import { formatErrorMessage } from "../../infra/errors.js"; import { redactSensitiveText } from "../../logging/redact.js"; -import type { WorkerLease, WorkerLeaseStatus, WorkerSshEndpoint } from "../../plugins/types.js"; -import { normalizeWorkerSshEndpoint } from "./store.js"; +import type { + WorkerDesktopEndpoint, + WorkerLease, + WorkerLeaseStatus, + WorkerSshEndpoint, +} from "../../plugins/types.js"; +import { normalizeWorkerDesktopEndpoint, normalizeWorkerSshEndpoint } from "./store.js"; export function requireWorkerLeaseStatus(value: unknown): WorkerLeaseStatus { if (!isRecord(value)) { @@ -39,6 +44,9 @@ export function requireWorkerLease(value: unknown): WorkerLease { leaseId: value.leaseId.trim(), ssh: normalizeWorkerSshEndpoint(value.ssh as WorkerSshEndpoint), ...(value.sharedHost === true ? { sharedHost: true } : {}), + ...(value.desktop === undefined + ? {} + : { desktop: normalizeWorkerDesktopEndpoint(value.desktop as WorkerDesktopEndpoint) }), }; } diff --git a/src/gateway/worker-environments/service.test.ts b/src/gateway/worker-environments/service.test.ts index 9ae4f44708d6..a1e93fbdc419 100644 --- a/src/gateway/worker-environments/service.test.ts +++ b/src/gateway/worker-environments/service.test.ts @@ -6,6 +6,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../../config/types.js"; import { WorkerProviderError, + type WorkerDesktopEndpoint, type WorkerProvider, type WorkerSshEndpoint, } from "../../plugins/types.js"; @@ -45,6 +46,11 @@ const SSH_ENDPOINT: WorkerSshEndpoint = { hostKey: HOST_KEY, keyRef: { source: "file", provider: "worker-keys", id: "/development-key" }, }; +const DESKTOP: WorkerDesktopEndpoint = { + protocol: "rfb", + port: 5900, + passwordFilePath: "/var/lib/crabbox/vnc.password", +}; const BUNDLE_HASH = "a".repeat(64); const BUNDLE_ARTIFACT: WorkerInstallationArtifact = { install: "bundle", @@ -102,6 +108,7 @@ describe("worker environment service", () => { store = createWorkerEnvironmentStore({ database, now: () => nowMs }); config = { cloudWorkers: { + desktop: true, profiles: { development: { provider: "fake", @@ -236,6 +243,37 @@ describe("worker environment service", () => { }); } + function seedReadyDesktop(environmentId: string) { + const intent = store.createIntent({ + environmentId, + providerId: "fake", + profileId: "development", + profileSnapshot: { settings: { region: "test", desktop: true } }, + provisionOperationId: `provision:${environmentId}`, + }); + const provisioning = store.transition({ + environmentId, + from: intent.state, + to: "provisioning", + }); + const bootstrapping = store.transition({ + environmentId, + from: provisioning.state, + to: "bootstrapping", + patch: { + leaseId: `lease:${environmentId}`, + sshEndpoint: SSH_ENDPOINT, + desktop: DESKTOP, + }, + }); + return store.transition({ + environmentId, + from: bootstrapping.state, + to: "ready", + patch: readyPatch(environmentId), + }); + } + function readyPatch(environmentId: string, receipt = BOOTSTRAP_RECEIPT) { return { bootstrapReceipt: receipt, @@ -1100,7 +1138,13 @@ describe("worker environment service", () => { stop: stopTunnel, stopAll: vi.fn(async () => {}), status: () => "stopped" as const, - } as WorkerTunnelManager; + desktop: { + acquire: vi.fn(), + attachObserver: vi.fn(), + stop: vi.fn(async () => {}), + stopAll: vi.fn(async () => {}), + }, + } as unknown as WorkerTunnelManager; const options = { generateWorkerCredential: () => [CREDENTIAL, String(++credentialSequence)].join("-"), tunnelManager, @@ -1599,6 +1643,25 @@ describe("worker environment service", () => { { leaseId: "lease-invalid", ssh: SSH_ENDPOINT, sharedHost: "yes" }, "invalid provision result", ], + [ + "unsupported desktop protocol", + { leaseId: "lease-invalid", ssh: SSH_ENDPOINT, desktop: { protocol: "rdp", port: 5900 } }, + 'desktop protocol must be "rfb"', + ], + [ + "invalid desktop port", + { leaseId: "lease-invalid", ssh: SSH_ENDPOINT, desktop: { protocol: "rfb", port: 0 } }, + "desktop port must be an integer", + ], + [ + "relative desktop password path", + { + leaseId: "lease-invalid", + ssh: SSH_ENDPOINT, + desktop: { protocol: "rfb", port: 5900, passwordFilePath: "vnc.password" }, + }, + "desktop password file path must be absolute", + ], ])("keeps %s from a provider retryable", async (_name, result, error) => { const workerService = createService(createProvider({ provision: async () => result as never })); @@ -2289,6 +2352,118 @@ describe("worker environment service", () => { expect(store.get("worker-isolation-change")?.sharedHost).toBe(true); }); + it("projects desktop availability only while a desktop lease is observable", () => { + const ready = seedReadyDesktop("worker-desktop-projection"); + const workerService = createService(createProvider()); + expect(workerService.get(ready.environmentId)).toMatchObject({ desktopAvailable: true }); + store.transition({ + environmentId: ready.environmentId, + from: ready.state, + to: "draining", + }); + expect(workerService.get(ready.environmentId)).toMatchObject({ desktopAvailable: false }); + }); + + it("acquires a desktop tunnel and mints a one-shot websocket path", async () => { + const record = seedReadyDesktop("worker-desktop-observe"); + const acquire = vi.fn(async () => ({ + localSocketPath: "/tmp/worker-desktop.sock", + vncPassword: "desktop-secret", + })); + const tunnelManager = { + desktop: { + acquire, + attachObserver: vi.fn(), + stop: vi.fn(async () => {}), + stopAll: vi.fn(async () => {}), + }, + status: () => "stopped" as const, + start: vi.fn(), + stop: vi.fn(async () => {}), + stopAll: vi.fn(async () => {}), + } as unknown as WorkerTunnelManager; + const workerService = createService(createProvider(), { tunnelManager }); + + await expect( + workerService.observeDesktop({ environmentId: record.environmentId, control: true }), + ).resolves.toMatchObject({ + transport: "rfb", + wsPath: expect.stringMatching(/^\/worker-desktop\/observe\?token=[a-f0-9]{48}$/u), + expiresAtMs: nowMs + 60_000, + control: true, + vncPassword: "desktop-secret", + }); + expect(acquire).toHaveBeenCalledWith( + expect.objectContaining({ + environmentId: record.environmentId, + ownerEpoch: record.ownerEpoch, + desktop: DESKTOP, + ssh: SSH_ENDPOINT, + resolveIdentity: expect.any(Function), + }), + ); + }); + + it("rejects desktop observe for invalid lifecycle gates and a stopped service", async () => { + const tunnelManager = { + desktop: { + acquire: vi.fn(), + attachObserver: vi.fn(), + stop: vi.fn(async () => {}), + stopAll: vi.fn(async () => {}), + }, + status: () => "stopped" as const, + start: vi.fn(), + stop: vi.fn(async () => {}), + stopAll: vi.fn(async () => {}), + } as unknown as WorkerTunnelManager; + const workerService = createService(createProvider(), { tunnelManager }); + const requested = store.createIntent({ + environmentId: "worker-desktop-requested", + providerId: "fake", + profileId: "development", + profileSnapshot: { settings: { region: "test" } }, + provisionOperationId: "provision:worker-desktop-requested", + }); + seedReady("worker-desktop-missing"); + const destroying = seedReadyDesktop("worker-desktop-destroy-requested"); + store.requestDestroy({ environmentId: destroying.environmentId, state: destroying.state }); + + config.cloudWorkers!.desktop = false; + await expect( + workerService.observeDesktop({ environmentId: requested.environmentId, control: false }), + ).rejects.toMatchObject({ + code: "invalid_state", + message: + "worker desktop observe is disabled; enable the Desktop lab in Control UI Settings -> Labs (config: cloudWorkers.desktop)", + }); + config.cloudWorkers!.desktop = true; + + for (const environmentId of [ + requested.environmentId, + "worker-desktop-missing", + destroying.environmentId, + ]) { + await expect( + workerService.observeDesktop({ environmentId, control: false }), + ).rejects.toMatchObject({ + code: "invalid_state", + message: "environment has no desktop; desktop is a warm-time capability of the profile", + }); + } + await expect( + workerService.observeDesktop({ environmentId: "worker-desktop-unknown", control: false }), + ).rejects.toMatchObject({ code: "environment_not_found" }); + await workerService.stop(); + await expect( + workerService.observeDesktop({ environmentId: destroying.environmentId, control: false }), + ).rejects.toMatchObject({ + code: "invalid_state", + message: "Worker environment service is stopping", + }); + expect(tunnelManager.desktop.acquire).not.toHaveBeenCalled(); + }); + it("fences a draining tunnel before reporting an unavailable provider", async () => { seedReady("worker-provider-missing"); const tunnelManager = { diff --git a/src/gateway/worker-environments/service.ts b/src/gateway/worker-environments/service.ts index 2391ecdc50d8..52a7f7ff6193 100644 --- a/src/gateway/worker-environments/service.ts +++ b/src/gateway/worker-environments/service.ts @@ -60,6 +60,7 @@ import { type WorkerInferenceSink, } from "./inference.js"; import type { WorkerLiveEventApplicationResult, WorkerLiveEventReceiver } from "./live-events.js"; +import type { WorkerDesktopObserveResult } from "./service-contract.js"; import { boundedWorkerError as boundedError, requireWorkerLeaseStatus, @@ -331,6 +332,7 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService ...((record.state === "failed" || record.state === "orphaned") && record.lastError ? { error: boundedError(record.lastError) } : {}), + desktopAvailable: inState(record, "ready", "idle", "attached") && record.desktop !== null, tunnelStatus: tunnels?.status(record.environmentId) ?? ("stopped" as const), }); @@ -643,6 +645,7 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService leaseId: lease.leaseId, sshEndpoint: lease.ssh, sharedHost: lease.sharedHost === true, + desktop: lease.desktop ?? null, }; const bootstrapping = move(record, "bootstrapping", patch); if (record.destroyRequestedAtMs !== null) { @@ -1165,6 +1168,79 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService } }; + const observeDesktop = async (request: { + environmentId: string; + control: boolean; + }): Promise => { + if (options.getConfig().cloudWorkers?.desktop !== true) { + throw serviceError( + "invalid_state", + "worker desktop observe is disabled; enable the Desktop lab in Control UI Settings -> Labs (config: cloudWorkers.desktop)", + ); + } + if (stopping) { + throw serviceError("invalid_state", "Worker environment service is stopping"); + } + if (!tunnels) { + throw serviceError("invalid_state", "Worker tunnel runtime is unavailable"); + } + let startup: Promise<{ localSocketPath: string; vncPassword?: string }> | undefined; + let ownerEpoch: number | undefined; + await withLock(request.environmentId, async () => { + if (stopping) { + throw serviceError("invalid_state", "Worker environment service is stopping"); + } + const record = store.get(request.environmentId); + if (!record) { + throw serviceError( + "environment_not_found", + `Unknown worker environment: ${request.environmentId}`, + ); + } + if ( + !inState(record, "ready", "idle", "attached") || + record.destroyRequestedAtMs !== null || + !record.leaseId || + !record.sshEndpoint || + !record.desktop + ) { + throw serviceError( + "invalid_state", + "environment has no desktop; desktop is a warm-time capability of the profile", + ); + } + const provider = providerFor(record.providerId); + ownerEpoch = record.ownerEpoch; + startup = tunnels.desktop.acquire({ + environmentId: record.environmentId, + ownerEpoch: record.ownerEpoch, + ssh: record.sshEndpoint, + desktop: record.desktop, + resolveIdentity: identityResolverFor(record, provider, record.leaseId), + }); + }); + if (!startup || ownerEpoch === undefined) { + throw serviceError("invalid_state", "Worker desktop tunnel failed to start"); + } + const acquired = await startup; + const { WORKER_DESKTOP_OBSERVE_PATH, mintWorkerDesktopObserverToken } = + await import("./desktop-observe.js"); + const minted = mintWorkerDesktopObserverToken({ + environmentId: request.environmentId, + ownerEpoch, + control: request.control, + localSocketPath: acquired.localSocketPath, + nowMs: now(), + }); + return { + transport: "rfb", + wsPath: `${WORKER_DESKTOP_OBSERVE_PATH}?token=${minted.token}`, + expiresAtMs: minted.expiresAtMs, + control: request.control, + ...(acquired.vncPassword ? { vncPassword: acquired.vncPassword } : {}), + }; + }; + const stopTunnel = async (environmentId: string, ownerEpoch?: number): Promise => { await withLock(environmentId, async () => { await tunnels?.stop(environmentId, ownerEpoch); @@ -1524,6 +1600,7 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService destroy: async (environmentId: string) => project(await destroy(environmentId)), destroyUnattached: async (environmentId: string) => project(await destroy(environmentId, { requireUnattached: true })), + observeDesktop, admitWorker: async (admission: WorkerConnectParams["admission"]) => { if (stopping) { return { ok: false, reason: "environment-unavailable" } as const; diff --git a/src/gateway/worker-environments/store.test.ts b/src/gateway/worker-environments/store.test.ts index 5bd65ea62f85..b16decf46079 100644 --- a/src/gateway/worker-environments/store.test.ts +++ b/src/gateway/worker-environments/store.test.ts @@ -4,7 +4,12 @@ import path from "node:path"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; import type { WorkerAdmissionHandshake } from "../../../packages/gateway-protocol/src/schema/worker-admission.js"; import { requireNodeSqlite } from "../../infra/node-sqlite.js"; -import type { WorkerProfile, WorkerSshEndpoint } from "../../plugins/types.js"; +import type { + WorkerDesktopEndpoint, + WorkerProfile, + WorkerSshEndpoint, +} from "../../plugins/types.js"; +import { ensureAdditiveStateColumns } from "../../state/openclaw-state-db-schema-additive.js"; import { assertOpenClawStateDatabaseForMaintenance, closeOpenClawStateDatabaseForTest, @@ -36,6 +41,11 @@ const SSH_ENDPOINT: WorkerEnvironmentSshEndpoint = { id: "/static-development-key", }, }; +const DESKTOP: WorkerDesktopEndpoint = { + protocol: "rfb", + port: 5900, + passwordFilePath: "/var/lib/crabbox/vnc.password", +}; const BOOTSTRAP_RECEIPT: WorkerEnvironmentBootstrapReceipt = { bundleHash: "a".repeat(64), openclawVersion: "2026.7.1", @@ -361,6 +371,58 @@ describe("worker environment store", () => { ).toThrow("SSH fallback ports"); }); + it("round-trips desktop metadata and clears it with the provider lease", () => { + createIntent("worker-desktop"); + store.transition({ + environmentId: "worker-desktop", + from: "requested", + to: "provisioning", + }); + store.transition({ + environmentId: "worker-desktop", + from: "provisioning", + to: "bootstrapping", + patch: { leaseId: "lease-desktop", sshEndpoint: SSH_ENDPOINT, desktop: DESKTOP }, + }); + closeOpenClawStateDatabaseForTest(); + database = openOpenClawStateDatabase({ env: { OPENCLAW_STATE_DIR: root } }); + store = createWorkerEnvironmentStore({ database, now: () => nowMs }); + expect(store.get("worker-desktop")?.desktop).toEqual(DESKTOP); + + const requested = store.requestDestroy({ + environmentId: "worker-desktop", + state: "bootstrapping", + terminalState: "failed", + }); + const draining = store.transition({ + environmentId: requested.environmentId, + from: requested.state, + to: "draining", + }); + const destroying = store.transition({ + environmentId: draining.environmentId, + from: draining.state, + to: "destroying", + }); + expect( + store.transition({ + environmentId: destroying.environmentId, + from: destroying.state, + to: "failed", + patch: { leaseId: null, sshEndpoint: null, lastError: "teardown complete" }, + }), + ).toMatchObject({ leaseId: null, sshEndpoint: null, desktop: null }); + }); + + it("idempotently ensures desktop_json on an existing state database", () => { + ensureAdditiveStateColumns(database.db); + ensureAdditiveStateColumns(database.db); + const columns = database.db.prepare("PRAGMA table_info(worker_environments)").all() as Array<{ + name: string; + }>; + expect(columns.filter((column) => column.name === "desktop_json")).toHaveLength(1); + }); + it("keeps renewal on one owner epoch and fences session replacement", () => { const bootstrapping = seedBootstrapping("worker-owner", "lease-owner"); store.transition({ diff --git a/src/gateway/worker-environments/store.ts b/src/gateway/worker-environments/store.ts index 5600fecb073f..2d0bb02fed09 100644 --- a/src/gateway/worker-environments/store.ts +++ b/src/gateway/worker-environments/store.ts @@ -1,4 +1,6 @@ +import { isAbsolute } from "node:path"; import type { DatabaseSync } from "node:sqlite"; +import { isRecord } from "@openclaw/normalization-core/record-coerce"; import { normalizeSortedUniqueTrimmedStringList } from "@openclaw/normalization-core/string-normalization"; import type { Insertable, Selectable, Updateable } from "kysely"; import { @@ -12,7 +14,11 @@ import { executeSqliteQueryTakeFirstSync, getNodeSqliteKysely, } from "../../infra/kysely-sync.js"; -import type { WorkerProfile, WorkerSshEndpoint } from "../../plugins/types.js"; +import type { + WorkerDesktopEndpoint, + WorkerProfile, + WorkerSshEndpoint, +} from "../../plugins/types.js"; import { isValidSecretRef } from "../../secrets/ref-contract.js"; import type { DB as StateDatabase, @@ -44,6 +50,7 @@ type RecordBase = RecordIdentity & { profileSnapshot: WorkerEnvironmentProfileSnapshot; provisionOperationId: string; sharedHost: boolean | null; + desktop: WorkerDesktopEndpoint | null; bootstrapReceipt: WorkerEnvironmentBootstrapReceipt | null; ownerEpoch: number; teardownTerminalState: WorkerEnvironmentTeardownTerminalState | null; @@ -69,6 +76,7 @@ export type WorkerEnvironmentTransitionPatch = { leaseId?: string | null; sshEndpoint?: WorkerEnvironmentSshEndpoint | null; sharedHost?: boolean; + desktop?: WorkerDesktopEndpoint | null; bootstrapReceipt?: WorkerEnvironmentBootstrapReceipt; attachedSessionIds?: readonly string[]; lastError?: string | null; @@ -273,6 +281,28 @@ export function normalizeWorkerSshEndpoint(value: Ssh): Ssh { keyRef: { ...value.keyRef }, }; } +export function normalizeWorkerDesktopEndpoint( + value: WorkerDesktopEndpoint, +): WorkerDesktopEndpoint { + if (!isRecord(value) || value.protocol !== "rfb") { + throw new Error('Worker environment desktop protocol must be "rfb"'); + } + if (!Number.isSafeInteger(value.port) || value.port < 1 || value.port > 65_535) { + throw new Error("Worker environment desktop port must be an integer from 1 through 65535"); + } + const passwordFilePath = value.passwordFilePath; + if ( + passwordFilePath !== undefined && + (typeof passwordFilePath !== "string" || !isAbsolute(passwordFilePath)) + ) { + throw new Error("Worker environment desktop password file path must be absolute"); + } + return { + protocol: "rfb", + port: value.port, + ...(passwordFilePath === undefined ? {} : { passwordFilePath }), + }; +} function endpointFrom(row: Row, fallbackPorts: readonly number[]): Ssh | null { const { ssh_host: host, @@ -293,6 +323,11 @@ function endpointFrom(row: Row, fallbackPorts: readonly number[]): Ssh | null { keyRef: JSON.parse(encoded) as Ssh["keyRef"], }); } +function desktopFrom(row: Row): WorkerDesktopEndpoint | null { + return row.desktop_json === null + ? null + : normalizeWorkerDesktopEndpoint(JSON.parse(row.desktop_json) as WorkerDesktopEndpoint); +} function bootstrapReceiptFrom(row: Row): WorkerEnvironmentBootstrapReceipt | null { const { bootstrap_bundle_hash: bundleHash, @@ -315,6 +350,7 @@ function assertShape( state: WorkerEnvironmentState, leaseId: string | null, sshEndpoint: Ssh | null, + desktop: WorkerDesktopEndpoint | null, bootstrapReceipt: WorkerEnvironmentBootstrapReceipt | null, attachedSessionIds: readonly string[], ): void { @@ -325,7 +361,7 @@ function assertShape( if (!sshEndpoint) { throw new Error("Worker environment provider lease requires an SSH endpoint reference"); } - } else if (leaseId || sshEndpoint) { + } else if (leaseId || sshEndpoint || desktop) { throw new Error(`Worker environment state ${state} cannot retain a provider lease`); } if (state === "bootstrapping" && bootstrapReceipt) { @@ -374,6 +410,7 @@ function fromRow(row: Row, fallbackPorts: readonly number[]): WorkerEnvironmentR sharedHost: row.shared_host === null ? null : row.shared_host === 1, leaseId: row.lease_id, sshEndpoint: endpointFrom(row, fallbackPorts), + desktop: desktopFrom(row), bootstrapReceipt: bootstrapReceiptFrom(row), ownerEpoch: row.owner_epoch, teardownTerminalState: teardownTerminalStateFrom(row.teardown_terminal_state), @@ -392,6 +429,7 @@ function fromRow(row: Row, fallbackPorts: readonly number[]): WorkerEnvironmentR record.state, record.leaseId, record.sshEndpoint, + record.desktop, record.bootstrapReceipt, record.attachedSessionIds, ); @@ -728,6 +766,7 @@ export function createWorkerEnvironmentStore( ssh_user: null, ssh_host_key: null, ssh_key_ref_json: null, + desktop_json: null, bootstrap_bundle_hash: null, bootstrap_openclaw_version: null, bootstrap_protocol_features_json: null, @@ -858,6 +897,14 @@ export function createWorkerEnvironmentStore( ? null : normalizeWorkerSshEndpoint(patch.sshEndpoint); const sharedHost = leaseId === null ? null : (patch.sharedHost ?? current.sharedHost); + const desktop = + leaseId === null + ? null + : patch.desktop === undefined + ? current.desktop + : patch.desktop === null + ? null + : normalizeWorkerDesktopEndpoint(patch.desktop); const acceptsBootstrapReceipt = from === "bootstrapping" && to === "ready"; if (patch.bootstrapReceipt !== undefined && !acceptsBootstrapReceipt) { throw new Error("Bootstrap receipt can only be recorded when a worker becomes ready"); @@ -895,7 +942,7 @@ export function createWorkerEnvironmentStore( : patch.attachedSessionIds === undefined ? current.attachedSessionIds : normalizeAttachedSessionIds(patch.attachedSessionIds); - assertShape(to, leaseId, sshEndpoint, bootstrapReceipt, attachedSessionIds); + assertShape(to, leaseId, sshEndpoint, desktop, bootstrapReceipt, attachedSessionIds); const [attachedSessionId] = attachedSessionIds; if (to === "attached" && attachedSessionId) { // Change session ownership atomically with worker state. @@ -941,6 +988,7 @@ export function createWorkerEnvironmentStore( ssh_user: sshEndpoint?.user ?? null, ssh_host_key: sshEndpoint?.hostKey ?? null, ssh_key_ref_json: sshEndpoint ? json(sshEndpoint.keyRef) : null, + desktop_json: desktop ? json(desktop) : null, bootstrap_bundle_hash: bootstrapReceipt?.bundleHash ?? null, bootstrap_openclaw_version: bootstrapReceipt?.openclawVersion ?? null, bootstrap_protocol_features_json: bootstrapReceipt diff --git a/src/gateway/worker-environments/tunnel.test.ts b/src/gateway/worker-environments/tunnel.test.ts index e019eb7014b1..2d9093f5c422 100644 --- a/src/gateway/worker-environments/tunnel.test.ts +++ b/src/gateway/worker-environments/tunnel.test.ts @@ -17,6 +17,37 @@ import { import { sshArgvPort } from "./worker-ssh-argv.test-support.js"; describe("worker tunnel manager", () => { + it("cascades only an epoch-matched environment stop into the desktop tunnel owner", async () => { + const fake = fakeRunner(); + const manager = createWorkerTunnelManager({ runner: fake.runner }); + const starting = manager.desktop.acquire({ + environmentId: "worker:desktop-cascade", + ownerEpoch: 2, + ssh: SSH, + desktop: { protocol: "rfb", port: 5900 }, + resolveIdentity, + }); + await waitForStarts(fake.starts, 1); + fake.starts[0]?.process.becomeReady(); + await starting; + const close = vi.fn(); + manager.desktop.attachObserver("worker:desktop-cascade", { + control: false, + ownerEpoch: 2, + close, + }); + + await manager.stop("worker:desktop-cascade", 1); + + expect(fake.starts[0]?.process.stopCount).toBe(0); + expect(close).not.toHaveBeenCalled(); + + await manager.stop("worker:desktop-cascade", 2); + + expect(fake.starts[0]?.process.stopCount).toBe(1); + expect(close).toHaveBeenCalledWith(1012, "desktop tunnel closed"); + }); + it("establishes a pinned reverse socket with keepalives and a separate workspace connection", async () => { const fake = fakeRunner(); const { manager, handle, start: tunnel } = await startConnectedTunnel(fake, "worker:one", 3); diff --git a/src/gateway/worker-environments/tunnel.ts b/src/gateway/worker-environments/tunnel.ts index 4a78630e28b6..6e7a479facd5 100644 --- a/src/gateway/worker-environments/tunnel.ts +++ b/src/gateway/worker-environments/tunnel.ts @@ -5,6 +5,7 @@ import { createSubsystemLogger } from "../../logging/subsystem.js"; import type { WorkerSshEndpoint } from "../../plugins/types.js"; import type { SpawnResult } from "../../process/exec.js"; import { createDeferred, type Deferred } from "../../shared/deferred.js"; +import { createWorkerDesktopTunnels } from "./desktop-tunnel.js"; import { boundedWorkerError } from "./service-validation.js"; import { advanceWorkerSshAfterTransportExit, @@ -144,6 +145,7 @@ export function createWorkerTunnelManager(options: WorkerTunnelManagerOptions = const backoff = options.backoff ?? DEFAULT_BACKOFF; const now = options.now ?? Date.now; const stableConnectionMs = options.stableConnectionMs ?? DEFAULT_STABLE_CONNECTION_MS; + const desktop = createWorkerDesktopTunnels({ runner, now }); const entries = new Map(); const claimedOwnerEpochs = new Map(); @@ -469,10 +471,10 @@ export function createWorkerTunnelManager(options: WorkerTunnelManagerOptions = async function stop(environmentId: string, ownerEpoch?: number): Promise { const entry = entries.get(environmentId); - if (!entry || (ownerEpoch !== undefined && ownerEpoch !== entry.ownerEpoch)) { - return; + if (entry && (ownerEpoch === undefined || ownerEpoch === entry.ownerEpoch)) { + await stopEntry(entry); } - await stopEntry(entry); + await desktop.stop(environmentId, ownerEpoch); } async function stopAll(): Promise { @@ -481,10 +483,11 @@ export function createWorkerTunnelManager(options: WorkerTunnelManagerOptions = entries.delete(entry.environmentId); entry.abortController.abort(new Error("Worker tunnel manager stopped")); } - await Promise.all(current.map(stopEntry)); + await Promise.all([...current.map(stopEntry), desktop.stopAll()]); } return { + desktop, start, stop, stopAll, diff --git a/src/gateway/worker-environments/worker-turn-launcher.test.ts b/src/gateway/worker-environments/worker-turn-launcher.test.ts index 3c8d042a476a..48487857b30e 100644 --- a/src/gateway/worker-environments/worker-turn-launcher.test.ts +++ b/src/gateway/worker-environments/worker-turn-launcher.test.ts @@ -272,6 +272,8 @@ describe("worker turn launcher", () => { destroyRequestedAtMs: null, tunnelStatus: "connected", state: "attached", + desktop: null, + desktopAvailable: false, leaseId: "lease-worker-turn", sshEndpoint: { host: "worker.example.test", diff --git a/src/plugin-sdk/plugin-entry.ts b/src/plugin-sdk/plugin-entry.ts index d71921e8aee0..24863a4c1889 100644 --- a/src/plugin-sdk/plugin-entry.ts +++ b/src/plugin-sdk/plugin-entry.ts @@ -131,6 +131,7 @@ export type { TranscriptSourceProvider, UnifiedModelCatalogProviderContext, UnifiedModelCatalogProviderPlugin, + WorkerDesktopEndpoint, WorkerLease, WorkerLeaseStatus, WorkerProfile, diff --git a/src/plugins/capability-provider.types.ts b/src/plugins/capability-provider.types.ts index d775e4f11c10..76b14ba37316 100644 --- a/src/plugins/capability-provider.types.ts +++ b/src/plugins/capability-provider.types.ts @@ -78,12 +78,23 @@ export type WorkerSshIdentityRequest = { keyRef: SecretRef; }; +/** Optional interactive desktop endpoint provisioned with the lease (warm-time capability). */ +export type WorkerDesktopEndpoint = { + /** Desktop service protocol on the worker loopback; "rfb" is the only phase-1 value. */ + protocol: "rfb"; + /** Loopback port on the worker (e.g. 5900). */ + port: number; + /** Absolute on-box path to the per-lease password file; read over SSH, never persisted as plaintext. */ + passwordFilePath?: string; +}; + /** Durable lease identity and endpoint returned by a successful provision operation. */ export type WorkerLease = { leaseId: string; ssh: WorkerSshEndpoint; /** The SSH account also owns processes unrelated to this worker lease. */ sharedHost?: boolean; + desktop?: WorkerDesktopEndpoint; }; /** Authoritative inspection result for an already-known worker lease. */ diff --git a/src/state/openclaw-database-maintenance.test.ts b/src/state/openclaw-database-maintenance.test.ts index 018fdd4ce97d..b06233a20f9c 100644 --- a/src/state/openclaw-database-maintenance.test.ts +++ b/src/state/openclaw-database-maintenance.test.ts @@ -137,6 +137,7 @@ describe("OpenClaw database maintenance schema validation", () => { ).toEqual([ "claw_installs.bootstrap_content_digest TEXT", "claw_installs.bootstrap_source_path TEXT", + "worker_environments.desktop_json TEXT", "claw_package_refs.extension_adapter_identity TEXT", "claw_package_refs.extension_detected_format TEXT", "claw_package_refs.extension_format TEXT", diff --git a/src/state/openclaw-state-db-additive-columns.ts b/src/state/openclaw-state-db-additive-columns.ts index c92e5605fe4a..b1a6e45b1be4 100644 --- a/src/state/openclaw-state-db-additive-columns.ts +++ b/src/state/openclaw-state-db-additive-columns.ts @@ -10,6 +10,7 @@ type LazyAdditiveStateColumnDefinition = { export const CLAW_LAZY_ADDITIVE_STATE_COLUMN_DEFINITIONS = [ { columnName: "bootstrap_content_digest", dataType: "TEXT", tableName: "claw_installs" }, { columnName: "bootstrap_source_path", dataType: "TEXT", tableName: "claw_installs" }, + { columnName: "desktop_json", dataType: "TEXT", tableName: "worker_environments" }, { columnName: "extension_adapter_identity", dataType: "TEXT", tableName: "claw_package_refs" }, { columnName: "extension_detected_format", dataType: "TEXT", tableName: "claw_package_refs" }, { columnName: "extension_format", dataType: "TEXT", tableName: "claw_package_refs" }, diff --git a/src/state/openclaw-state-db-maintenance.ts b/src/state/openclaw-state-db-maintenance.ts index 64206d80680b..8d060190814b 100644 --- a/src/state/openclaw-state-db-maintenance.ts +++ b/src/state/openclaw-state-db-maintenance.ts @@ -19,13 +19,14 @@ import { resolveOpenClawStateSqlitePath } from "./openclaw-state-db.paths.js"; import { OPENCLAW_STATE_SCHEMA_SQL } from "./openclaw-state-schema.js"; /** - * Additive Claw provenance columns that only a writable open can ensure. A - * same-version database written before them stays readable so read-only - * planning surfaces are not refused before they can report anything. + * Additive Claw provenance and worker-environment columns that only a writable + * open can ensure. A same-version database written before them stays readable + * so read-only planning surfaces are not refused before they can report anything. */ export const CLAW_LAZY_ADDITIVE_STATE_COLUMNS = [ "claw_installs.bootstrap_content_digest", "claw_installs.bootstrap_source_path", + "worker_environments.desktop_json", "claw_package_refs.extension_adapter_identity", "claw_package_refs.extension_detected_format", "claw_package_refs.extension_format", @@ -70,6 +71,7 @@ const OPENCLAW_STATE_MAINTENANCE_SCHEMA_COMPATIBILITY = { "target_agent_id TEXT NOT NULL DEFAULT 'main'", ], "operator_approvals.resolution_ref": ["resolution_ref TEXT"], + "worker_environments.desktop_json": ["desktop_json TEXT"], "worker_environments.shared_host": ["shared_host INTEGER CHECK (shared_host IN (0, 1))"], }, } satisfies SqliteSchemaCompatibility; diff --git a/src/state/openclaw-state-db.generated.d.ts b/src/state/openclaw-state-db.generated.d.ts index 6d851e163888..50195dde759b 100644 --- a/src/state/openclaw-state-db.generated.d.ts +++ b/src/state/openclaw-state-db.generated.d.ts @@ -1421,6 +1421,7 @@ export interface WorkerEnvironments { bootstrap_openclaw_version: string | null; bootstrap_protocol_features_json: string | null; created_at_ms: number; + desktop_json: string | null; destroy_requested_at_ms: number | null; environment_id: string; idle_since_at_ms: number | null; diff --git a/src/state/openclaw-state-db.test.ts b/src/state/openclaw-state-db.test.ts index 03a25de98295..ea1c8b6893f9 100644 --- a/src/state/openclaw-state-db.test.ts +++ b/src/state/openclaw-state-db.test.ts @@ -1958,6 +1958,7 @@ INSERT INTO macos_port_guardian_records VALUES (4242, 18789, '/usr/bin/ssh', 're it.each([ { columnName: "run_end_cleanup_json", tableName: "worktrees" }, + { columnName: "desktop_json", tableName: "worker_environments" }, { columnName: "shared_host", tableName: "worker_environments" }, ])( "appends same-version $columnName to $tableName before schema validation", @@ -2809,6 +2810,22 @@ INSERT INTO macos_port_guardian_records VALUES (4242, 18789, '/usr/bin/ssh', 're ); }); + it("opens a pre-desktop current-schema database read-only", async () => { + const stateDir = createTempStateDir(); + const databasePath = materializeCurrentStateDatabase(stateDir); + const { DatabaseSync } = requireNodeSqlite(); + const preDesktop = new DatabaseSync(databasePath); + try { + preDesktop.exec("ALTER TABLE worker_environments DROP COLUMN desktop_json;"); + } finally { + preDesktop.close(); + } + + const database = await openExistingOpenClawStateDatabaseReadOnly({ path: databasePath }); + expect(database).toBeDefined(); + database?.walMaintenance.close(); + }); + it("reports success when retrying transient read-only snapshot cleanup", async () => { const stateDir = createTempStateDir(); const databasePath = materializeCurrentStateDatabase(stateDir); diff --git a/src/state/openclaw-state-schema.sql b/src/state/openclaw-state-schema.sql index ba8d27afaec4..764bda0c3efd 100644 --- a/src/state/openclaw-state-schema.sql +++ b/src/state/openclaw-state-schema.sql @@ -1851,6 +1851,7 @@ CREATE TABLE IF NOT EXISTS worker_environments ( ssh_user TEXT, ssh_host_key TEXT, ssh_key_ref_json TEXT, + desktop_json TEXT, state TEXT NOT NULL CHECK ( state IN ( 'requested', diff --git a/ui/package.json b/ui/package.json index 1766fca4d225..6fe13757c756 100644 --- a/ui/package.json +++ b/ui/package.json @@ -21,6 +21,7 @@ "@modelcontextprotocol/ext-apps": "1.7.5", "@modelcontextprotocol/sdk": "1.30.0", "@noble/ed25519": "3.1.0", + "@novnc/novnc": "^1.7.0", "@openclaw/gateway-client": "workspace:*", "@openclaw/gateway-protocol": "workspace:*", "@openclaw/libterminal": "0.3.2", diff --git a/ui/src/app/app-host.ts b/ui/src/app/app-host.ts index a4c52e5030b9..830562322890 100644 --- a/ui/src/app/app-host.ts +++ b/ui/src/app/app-host.ts @@ -44,6 +44,7 @@ import { selectShellRouteState, type ShellRouteState } from "./app-host-route-st import { OpenClawApp } from "./app-root.ts"; import { isBrowserPanelAvailable, + isDesktopPanelAvailable, ShellChromeOwner, type ShellChromeHost, } from "./app-shell-chrome.ts"; @@ -62,6 +63,7 @@ import { BROWSER_PANEL_ELEMENT, COMMAND_PALETTE_ELEMENT, CUSTODIAN_PANEL_ELEMENT, + DESKTOP_PANEL_ELEMENT, EXEC_APPROVAL_ELEMENT, preloadOptionalElement, TERMINAL_PANEL_ELEMENT, @@ -130,6 +132,7 @@ class OpenClawShell readonly commandPaletteElement = COMMAND_PALETTE_ELEMENT; readonly terminalPanelElement = TERMINAL_PANEL_ELEMENT; readonly browserPanelElement = BROWSER_PANEL_ELEMENT; + readonly desktopPanelElement = DESKTOP_PANEL_ELEMENT; readonly custodianPanelElement = CUSTODIAN_PANEL_ELEMENT; readonly execApprovalElement = EXEC_APPROVAL_ELEMENT; @query("openclaw-command-palette") commandPalette: CommandPaletteElement | undefined; @@ -526,6 +529,9 @@ class OpenClawShell if (isBrowserPanelAvailable(gatewaySnapshot)) { preloadOptionalElement(this, this.browserPanelElement); } + if (isDesktopPanelAvailable(gatewaySnapshot)) { + preloadOptionalElement(this, this.desktopPanelElement); + } if (isGatewayMethodAdvertised(gatewaySnapshot, "openclaw.chat") === true) { preloadOptionalElement(this, this.custodianPanelElement); } diff --git a/ui/src/app/app-shell-chrome.ts b/ui/src/app/app-shell-chrome.ts index 212eb9f80645..7307f77d4b12 100644 --- a/ui/src/app/app-shell-chrome.ts +++ b/ui/src/app/app-shell-chrome.ts @@ -13,6 +13,7 @@ import type { OpenClawModalDialog } from "../components/modal-dialog.ts"; import { BROWSER_PANEL_TOGGLE_EVENT, CUSTODIAN_PANEL_TOGGLE_EVENT, + DESKTOP_PANEL_TOGGLE_EVENT, isTerminalPanelShortcut, TERMINAL_PANEL_TOGGLE_EVENT, type PanelToggleElement, @@ -52,6 +53,16 @@ export function isBrowserPanelAvailable( ); } +export function isDesktopPanelAvailable( + snapshot: ApplicationContext["gateway"]["snapshot"], +): boolean { + return ( + snapshot.phase === "connected" && + hasOperatorAdminAccess(snapshot.hello?.auth ?? null) && + isGatewayMethodAdvertised(snapshot, "worker.desktop.observe") === true + ); +} + export interface ShellChromeHost extends HTMLElement { readonly context: ApplicationContext | undefined; readonly onboardingMode: boolean; @@ -59,6 +70,7 @@ export interface ShellChromeHost extends HTMLElement { readonly commandPaletteElement: OptionalCustomElement; readonly terminalPanelElement: OptionalCustomElement; readonly browserPanelElement: OptionalCustomElement; + readonly desktopPanelElement: OptionalCustomElement; readonly custodianPanelElement: OptionalCustomElement; readonly execApprovalElement: OptionalCustomElement; readonly commandPalette: CommandPaletteElement | undefined; @@ -103,6 +115,7 @@ export class ShellChromeOwner { window.addEventListener("openclaw:native-navigate", this.handleNativeNavigate); window.addEventListener(TERMINAL_PANEL_TOGGLE_EVENT, this.handleDeferredTerminalToggle); window.addEventListener(BROWSER_PANEL_TOGGLE_EVENT, this.handleDeferredBrowserToggle); + window.addEventListener(DESKTOP_PANEL_TOGGLE_EVENT, this.handleDeferredDesktopToggle); window.addEventListener(CUSTODIAN_PANEL_TOGGLE_EVENT, this.handleDeferredCustodianToggle); } @@ -123,6 +136,7 @@ export class ShellChromeOwner { window.removeEventListener("openclaw:native-navigate", this.handleNativeNavigate); window.removeEventListener(TERMINAL_PANEL_TOGGLE_EVENT, this.handleDeferredTerminalToggle); window.removeEventListener(BROWSER_PANEL_TOGGLE_EVENT, this.handleDeferredBrowserToggle); + window.removeEventListener(DESKTOP_PANEL_TOGGLE_EVENT, this.handleDeferredDesktopToggle); window.removeEventListener(CUSTODIAN_PANEL_TOGGLE_EVENT, this.handleDeferredCustodianToggle); } @@ -485,6 +499,17 @@ export class ShellChromeOwner { } }; + readonly handleDeferredDesktopToggle = (event: Event): void => { + const host = this.host; + if (isOptionalElementDefined(host.desktopPanelElement)) { + return; + } + const snapshot = host.context?.gateway?.snapshot; + if (snapshot && isDesktopPanelAvailable(snapshot)) { + this.deliverPanelEventAfterLoad(host.desktopPanelElement, event); + } + }; + readonly handleDeferredCustodianToggle = (event: Event): void => { const host = this.host; if (isOptionalElementDefined(host.custodianPanelElement)) { diff --git a/ui/src/app/app-shell-view.ts b/ui/src/app/app-shell-view.ts index 8b3d82492f03..dabc7b3399d0 100644 --- a/ui/src/app/app-shell-view.ts +++ b/ui/src/app/app-shell-view.ts @@ -20,7 +20,7 @@ import type { NewSessionTarget } from "../pages/new-session/location.ts"; import { pluginTabKey, pluginTabRefFromSearch } from "../pages/plugin/route.ts"; import type { ShellRouteState } from "./app-host-route-state.ts"; import { resolveTerminalThemeMode } from "./app-root.ts"; -import { isBrowserPanelAvailable } from "./app-shell-chrome.ts"; +import { isBrowserPanelAvailable, isDesktopPanelAvailable } from "./app-shell-chrome.ts"; import type { OutboxStoreRuntime, StoredOutboxScopeHost } from "./app-shell-gateway.ts"; import { findInlineApproval } from "./approval-presentation.ts"; import type { ApplicationRuntime } from "./bootstrap.ts"; @@ -115,6 +115,7 @@ export function renderApplicationShell(host: ShellViewHost) { context.config.current.terminalEnabled ?? false, ); const browserPanelAvailable = isBrowserPanelAvailable(gatewaySnapshot); + const desktopPanelAvailable = isDesktopPanelAvailable(gatewaySnapshot); const custodianPanelAvailable = gatewayConnected && isGatewayMethodAdvertised(gatewaySnapshot, "openclaw.chat") === true; const activeRoute = host.routeState.routeId ?? "chat"; @@ -477,6 +478,11 @@ export function renderApplicationShell(host: ShellViewHost) { password: context.gateway.connection.password, })} > + import("../components/browser/browser-panel.ts"), } satisfies OptionalCustomElement; +export const DESKTOP_PANEL_ELEMENT = { + tagName: "openclaw-desktop-panel", + label: "desktop panel", + loadModule: () => import("../components/desktop/desktop-panel.ts"), +} satisfies OptionalCustomElement; + export const CUSTODIAN_PANEL_ELEMENT = { tagName: "openclaw-custodian-panel", label: "custodian panel", diff --git a/ui/src/components/command-palette.ts b/ui/src/components/command-palette.ts index ffa71c755274..941a95ddfab9 100644 --- a/ui/src/components/command-palette.ts +++ b/ui/src/components/command-palette.ts @@ -4,6 +4,7 @@ import { html, nothing } from "lit"; import { property, state } from "lit/decorators.js"; import { ref } from "lit/directives/ref.js"; import type { RouteId } from "../app-route-paths.ts"; +import { isDesktopPanelAvailable } from "../app/app-shell-chrome.ts"; import { applicationContext, type ApplicationContext } from "../app/context.ts"; import { t } from "../i18n/index.ts"; import { formatRelativeTimestamp } from "../lib/format.ts"; @@ -14,6 +15,7 @@ import { OpenClawLightDomContentsElement } from "../lit/openclaw-element.ts"; import { SubscriptionsController } from "../lit/subscriptions-controller.ts"; import { isCommandPaletteShortcut } from "./command-palette-contract.ts"; import { icons, type IconName } from "./icons.ts"; +import { DESKTOP_PANEL_TOGGLE_EVENT } from "./panel-toggle-contract.ts"; import "./modal-dialog.ts"; type PaletteItem = { @@ -31,7 +33,7 @@ const SESSION_SEARCH_LIMIT = 10; const SESSION_SEARCH_MAX_PAGES = 4; const SESSION_SEARCH_PAGE_SIZE = 50; -function getPaletteBaseItems(): PaletteItem[] { +function getPaletteBaseItems(desktopAvailable: boolean): PaletteItem[] { return [ { id: "nav-new-session", @@ -97,11 +99,22 @@ function getPaletteBaseItems(): PaletteItem[] { action: "/verbose full", description: t("palette.descriptions.verboseMode"), }, + ...(desktopAvailable + ? [ + { + id: "panel-desktop", + label: t("palette.items.desktop"), + icon: "monitor" as const, + category: "navigation" as const, + action: "panel:desktop", + }, + ] + : []), ]; } -function getPaletteItemsInternal(): PaletteItem[] { - return getPaletteBaseItems(); +function getPaletteItemsInternal(desktopAvailable: boolean): PaletteItem[] { + return getPaletteBaseItems(desktopAvailable); } type CommandPaletteProps = { @@ -115,6 +128,7 @@ type CommandPaletteProps = { onNavigate: (routeId: RouteId) => void; onSelectSession?: (sessionKey: string) => void; onSlashCommand?: (command: string) => void; + desktopAvailable: boolean; onInputRef: (element: Element | undefined) => void; }; @@ -122,8 +136,9 @@ function filteredItems( query: string, includeSlashCommands = true, sessionItems: readonly PaletteItem[] = [], + desktopAvailable = false, ): PaletteItem[] { - const items = getPaletteItemsInternal().filter( + const items = getPaletteItemsInternal(desktopAvailable).filter( (item) => includeSlashCommands || item.category !== "search", ); if (!query) { @@ -159,6 +174,8 @@ function selectItem(item: PaletteItem, props: CommandPaletteProps) { props.onNavigate(item.action.slice(4) as RouteId); } else if (item.action.startsWith(SESSION_ACTION_PREFIX)) { props.onSelectSession?.(item.action.slice(SESSION_ACTION_PREFIX.length)); + } else if (item.action === "panel:desktop") { + window.dispatchEvent(new CustomEvent(DESKTOP_PANEL_TOGGLE_EVENT, { detail: { open: true } })); } else { props.onSlashCommand?.(item.action); } @@ -177,7 +194,12 @@ function scrollActiveIntoView() { } function handleKeydown(e: KeyboardEvent, props: CommandPaletteProps) { - const items = filteredItems(props.query, Boolean(props.onSlashCommand), props.sessionItems); + const items = filteredItems( + props.query, + Boolean(props.onSlashCommand), + props.sessionItems, + props.desktopAvailable, + ); if (items.length === 0 && (e.key === "ArrowDown" || e.key === "ArrowUp" || e.key === "Enter")) { return; } @@ -242,7 +264,12 @@ function renderCommandPalette(props: CommandPaletteProps) { if (!props.open) { return nothing; } - const items = filteredItems(props.query, Boolean(props.onSlashCommand), props.sessionItems); + const items = filteredItems( + props.query, + Boolean(props.onSlashCommand), + props.sessionItems, + props.desktopAvailable, + ); const grouped = groupItems(items); const activeItem = items[props.activeIndex]; const activeOptionId = activeItem ? getOptionId(activeItem) : nothing; @@ -543,6 +570,9 @@ export class CommandPalette extends OpenClawLightDomContentsElement { query: this.query, activeIndex: this.activeIndex, sessionItems: this.sessionItems, + desktopAvailable: this.context + ? isDesktopPanelAvailable(this.context.gateway.snapshot) + : false, onToggle: this.togglePalette, onQueryChange: (query) => { this.query = query; diff --git a/ui/src/components/desktop/desktop-client.test.ts b/ui/src/components/desktop/desktop-client.test.ts new file mode 100644 index 000000000000..46e9ed99c0a2 --- /dev/null +++ b/ui/src/components/desktop/desktop-client.test.ts @@ -0,0 +1,108 @@ +/* @vitest-environment jsdom */ + +import { describe, expect, it, vi } from "vitest"; +import { DesktopClient } from "./desktop-client.ts"; + +type RfbConstructor = NonNullable[0]>; +type RfbClient = InstanceType; + +class FakeSocket extends EventTarget { + readonly url: string; + + constructor(url: string) { + super(); + this.url = url; + } +} + +function createFakeRfb() { + const instances: FakeRfb[] = []; + class FakeRfb extends EventTarget implements RfbClient { + viewOnly = false; + scaleViewport = false; + readonly disconnect = vi.fn(); + + constructor( + readonly target: HTMLElement, + readonly channel: string | WebSocket, + readonly options?: { credentials?: { password: string } }, + ) { + super(); + instances.push(this); + } + } + return { Rfb: FakeRfb as RfbConstructor, instances }; +} + +describe("DesktopClient", () => { + it.each([ + [ + "http://control.example.test/chat", + "ws://control.example.test/worker-desktop/observe?token=abc", + ], + [ + "https://control.example.test/chat", + "wss://control.example.test/worker-desktop/observe?token=abc", + ], + ])("resolves relative observer URLs against %s", async (gatewayUrl, expectedUrl) => { + const { Rfb, instances } = createFakeRfb(); + const sockets: FakeSocket[] = []; + const client = new DesktopClient(Rfb, (url) => { + const socket = new FakeSocket(url); + sockets.push(socket); + return socket as unknown as WebSocket; + }); + const target = document.createElement("div"); + + await client.connect({ + gatewayUrl, + wsUrl: "/worker-desktop/observe?token=abc", + password: "secret", + viewOnly: true, + target, + }); + + expect(sockets[0]?.url).toBe(expectedUrl); + expect(instances[0]?.target).toBe(target); + expect(instances[0]?.channel).toBe(sockets[0]); + }); + + it("propagates RFB options and disconnects through the returned handle", async () => { + const { Rfb, instances } = createFakeRfb(); + const socket = new FakeSocket("ws://control.example.test/worker-desktop/observe"); + const client = new DesktopClient(Rfb, () => socket as unknown as WebSocket); + + const handle = await client.connect({ + gatewayUrl: "ws://control.example.test", + wsUrl: "/worker-desktop/observe", + password: "secret", + viewOnly: false, + target: document.createElement("div"), + }); + + expect(instances[0]?.viewOnly).toBe(false); + expect(instances[0]?.scaleViewport).toBe(true); + expect(instances[0]?.options).toEqual({ credentials: { password: "secret" } }); + + handle.disconnect(); + expect(instances[0]?.disconnect).toHaveBeenCalledOnce(); + }); + + it("forwards socket close metadata through the RFB disconnect callback", async () => { + const { Rfb, instances } = createFakeRfb(); + const socket = new FakeSocket("ws://control.example.test/worker-desktop/observe"); + const onDisconnect = vi.fn(); + const client = new DesktopClient(Rfb, () => socket as unknown as WebSocket); + + await client.connect({ + wsUrl: "ws://control.example.test/worker-desktop/observe", + viewOnly: true, + target: document.createElement("div"), + onDisconnect, + }); + socket.dispatchEvent(new CloseEvent("close", { code: 4000, reason: "control-taken" })); + instances[0]?.dispatchEvent(new CustomEvent("disconnect", { detail: { clean: true } })); + + expect(onDisconnect).toHaveBeenCalledWith({ code: 4000, reason: "control-taken" }); + }); +}); diff --git a/ui/src/components/desktop/desktop-client.ts b/ui/src/components/desktop/desktop-client.ts new file mode 100644 index 000000000000..577919f4b3b8 --- /dev/null +++ b/ui/src/components/desktop/desktop-client.ts @@ -0,0 +1,100 @@ +type DesktopDisconnectDetail = { + code?: number; + reason?: string; +}; + +type DesktopSecurityFailureDetail = { + reason?: string; + status?: number; +}; + +export type DesktopConnectOptions = { + gatewayUrl?: string; + onConnect?: () => void; + onDisconnect?: (detail: DesktopDisconnectDetail) => void; + onSecurityFailure?: (detail: DesktopSecurityFailureDetail) => void; + password?: string; + target: HTMLElement; + viewOnly: boolean; + wsUrl: string; +}; + +export type DesktopConnectionHandle = { + disconnect(): void; +}; + +type RfbClient = EventTarget & { + disconnect(): void; + scaleViewport: boolean; + viewOnly: boolean; +}; + +type RfbConstructor = new ( + target: HTMLElement, + channel: string | WebSocket, + options?: { credentials?: { password: string } }, +) => RfbClient; + +type RfbLoader = () => Promise; +type WebSocketFactory = (url: string) => WebSocket; + +const loadDefaultRfb: RfbLoader = async () => { + // @novnc/novnc 1.7 exports RFB from the package root; keeping this import + // here ensures the substantial client stays in the lazy desktop chunk. + const module = (await import("@novnc/novnc")) as { default: RfbConstructor }; + return module.default; +}; + +function resolveDesktopWebSocketUrl(wsUrl: string, gatewayUrl = globalThis.location?.href): string { + const base = new URL(gatewayUrl ?? globalThis.location.href, globalThis.location?.href); + if (base.protocol === "http:") { + base.protocol = "ws:"; + } else if (base.protocol === "https:") { + base.protocol = "wss:"; + } + const resolved = new URL(wsUrl, base); + if (resolved.protocol === "http:") { + resolved.protocol = "ws:"; + } else if (resolved.protocol === "https:") { + resolved.protocol = "wss:"; + } + if (resolved.protocol !== "ws:" && resolved.protocol !== "wss:") { + throw new Error("Desktop observer URL must use WebSocket transport"); + } + return resolved.toString(); +} + +/** Thin owner for one noVNC RFB lifecycle. */ +export class DesktopClient { + constructor( + private readonly rfbConstructor?: RfbConstructor, + private readonly createWebSocket: WebSocketFactory = (url) => new WebSocket(url), + private readonly loadRfb: RfbLoader = loadDefaultRfb, + ) {} + + async connect(options: DesktopConnectOptions): Promise { + const Rfb = this.rfbConstructor ?? (await this.loadRfb()); + const wsUrl = resolveDesktopWebSocketUrl(options.wsUrl, options.gatewayUrl); + const socket = this.createWebSocket(wsUrl); + let closeDetail: DesktopDisconnectDetail = {}; + socket.addEventListener("close", (event) => { + closeDetail = { code: event.code, reason: event.reason }; + }); + const rfb = new Rfb( + options.target, + socket, + options.password ? { credentials: { password: options.password } } : undefined, + ); + rfb.viewOnly = options.viewOnly; + rfb.scaleViewport = true; + rfb.addEventListener("connect", () => options.onConnect?.()); + rfb.addEventListener("disconnect", () => options.onDisconnect?.(closeDetail)); + rfb.addEventListener("securityfailure", (event) => { + const detail = (event as CustomEvent).detail ?? {}; + options.onSecurityFailure?.(detail); + }); + return { + disconnect: () => rfb.disconnect(), + }; + } +} diff --git a/ui/src/components/desktop/desktop-panel.ts b/ui/src/components/desktop/desktop-panel.ts new file mode 100644 index 000000000000..f79fb6e043eb --- /dev/null +++ b/ui/src/components/desktop/desktop-panel.ts @@ -0,0 +1,584 @@ +import type { + EnvironmentSummary, + EnvironmentsListResult, + WorkerDesktopObserveResult, +} from "@openclaw/gateway-protocol"; +import { css, html, nothing, svg } from "lit"; +import { property, state } from "lit/decorators.js"; +import type { GatewayBrowserClient } from "../../api/gateway.ts"; +import { t } from "../../i18n/index.ts"; +import { formatUiError } from "../../lib/format-error.ts"; +import { OpenClawLitElement } from "../../lit/openclaw-element.ts"; +import { DockLayoutController, dockPanelStyles } from "../dock-layout-controller.ts"; +import { createDockPanelLayout } from "../dock-panel-layout.ts"; +import { + DESKTOP_PANEL_TOGGLE_EVENT, + type DesktopPanelToggleDetail, +} from "../panel-toggle-contract.ts"; +import { + DesktopClient, + type DesktopConnectionHandle, + type DesktopConnectOptions, +} from "./desktop-client.ts"; + +const CLOSE_GLYPH = svg``; +const DOCK_BOTTOM_GLYPH = svg``; +const DOCK_RIGHT_GLYPH = svg``; + +const panelLayout = createDockPanelLayout({ + storageKey: "openclaw.desktopPanel", + minHeight: 240, + minWidth: 380, + defaultDock: "right", + supportedDocks: ["bottom", "right"], + defaultHeight: 420, + defaultWidth: 560, +}); + +type DesktopPanelState = "picker" | "connecting" | "connected" | "disconnected"; +type DesktopClientFactory = () => { + connect(options: DesktopConnectOptions): Promise; +}; + +/** `` — dockable RFB access to cloud-worker desktops. */ +class OpenClawDesktopPanel extends OpenClawLitElement { + @property({ attribute: false }) client: GatewayBrowserClient | null = null; + @property({ type: Boolean }) available = false; + @property({ type: Boolean }) suppressed = false; + + /** Browser tests replace the transport without opening a real RFB socket. */ + desktopClientFactory: DesktopClientFactory = () => new DesktopClient(); + + @state() private environments: EnvironmentSummary[] = []; + @state() private loading = false; + @state() private state: DesktopPanelState = "picker"; + @state() private environmentId: string | null = null; + @state() private controlling = false; + @state() private errorText: string | null = null; + @state() private noticeText: string | null = null; + @state() private disconnectedReason: string | null = null; + + private connection: DesktopConnectionHandle | null = null; + private operationId = 0; + private controlTakeoverRecoveryUsed = false; + private readonly dockLayout = new DockLayoutController(this, { + layout: panelLayout, + reservationPrefix: "desktop", + isAvailable: () => this.available, + }); + private readonly onToggleRequest = (event: Event) => this.handleToggleRequest(event); + + static override styles = [ + dockPanelStyles, + css` + .bp--bottom { + left: var(--shell-nav-width, 0); + right: calc(var(--oc-terminal-reserve-right, 0px) + var(--oc-browser-reserve-right, 0px)); + bottom: calc( + var(--oc-terminal-reserve-bottom, 0px) + var(--oc-browser-reserve-bottom, 0px) + ); + } + .bp--right { + top: var(--shell-topbar-height, 0); + right: calc(var(--oc-terminal-reserve-right, 0px) + var(--oc-browser-reserve-right, 0px)); + bottom: var(--oc-terminal-reserve-bottom, 0px); + } + .bp-title { + min-width: 0; + padding-left: 8px; + font-size: 13px; + font-weight: 600; + } + .bp-icon.is-active { + color: var(--accent, #ff5c5c); + background: color-mix(in srgb, var(--accent, #ff5c5c) 14%, transparent); + } + .desktop-content { + display: flex; + flex: 1; + min-height: 0; + flex-direction: column; + } + .desktop-toolbar { + display: flex; + align-items: center; + gap: 8px; + padding: 8px 10px; + border-bottom: 1px solid var(--border, #262b34); + } + .desktop-toolbar__spacer { + flex: 1; + } + .desktop-button { + border: 1px solid var(--border, #262b34); + border-radius: 6px; + padding: 5px 10px; + background: transparent; + color: var(--text, #d7dae0); + font: inherit; + font-size: 12px; + } + .desktop-button:hover:not(:disabled) { + background: color-mix(in srgb, var(--text, #d7dae0) 10%, transparent); + } + .desktop-button--primary { + border-color: var(--accent, #ff5c5c); + color: var(--accent, #ff5c5c); + } + .desktop-button:disabled { + opacity: 0.5; + } + .desktop-badge, + .desktop-session { + border-radius: 999px; + padding: 2px 8px; + background: color-mix(in srgb, var(--text, #d7dae0) 10%, transparent); + color: var(--muted, #8a919e); + font-size: 11px; + } + .desktop-badge--control { + color: var(--accent, #ff5c5c); + background: color-mix(in srgb, var(--accent, #ff5c5c) 12%, transparent); + } + .desktop-note { + padding: 7px 12px; + border-bottom: 1px solid var(--border, #262b34); + color: var(--muted, #8a919e); + font-size: 12px; + } + .desktop-note--error { + color: var(--danger, #ff6b6b); + } + .desktop-picker, + .desktop-status { + display: flex; + flex: 1; + min-height: 0; + flex-direction: column; + gap: 10px; + overflow: auto; + padding: 14px; + } + .desktop-status { + align-items: center; + justify-content: center; + text-align: center; + color: var(--muted, #8a919e); + } + .desktop-environment { + display: flex; + align-items: center; + gap: 10px; + padding: 10px; + border: 1px solid var(--border, #262b34); + border-radius: 8px; + } + .desktop-environment__details { + display: flex; + flex: 1; + min-width: 0; + flex-direction: column; + gap: 5px; + } + .desktop-environment__id { + overflow: hidden; + color: var(--text, #d7dae0); + font-family: ui-monospace, SFMono-Regular, Menlo, monospace; + font-size: 12px; + text-overflow: ellipsis; + white-space: nowrap; + } + .desktop-environment__meta, + .desktop-environment__sessions { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 5px; + color: var(--muted, #8a919e); + font-size: 11px; + } + .desktop-surface { + flex: 1; + min-height: 0; + overflow: hidden; + background: #202020; + } + `, + ]; + + override connectedCallback(): void { + super.connectedCallback(); + window.addEventListener(DESKTOP_PANEL_TOGGLE_EVENT, this.onToggleRequest); + this.dockLayout.setSuppressed(this.suppressed); + if (this.dockLayout.open) { + void this.refreshEnvironments(); + } + } + + override disconnectedCallback(): void { + window.removeEventListener(DESKTOP_PANEL_TOGGLE_EVENT, this.onToggleRequest); + this.disconnectConnection(); + super.disconnectedCallback(); + } + + override updated(changed: Map): void { + if (changed.has("suppressed")) { + const restored = this.dockLayout.setSuppressed(this.suppressed); + if (this.suppressed) { + this.returnToPicker(); + } else if (restored) { + void this.refreshEnvironments(); + } + } + if (changed.has("client") || changed.has("available")) { + if (!this.available && this.dockLayout.open) { + this.dockLayout.hideWithoutPersisting(); + this.returnToPicker(); + } else if (this.available && this.dockLayout.restoreOpenState()) { + void this.refreshEnvironments(); + } + } + this.dockLayout.syncReservation(); + } + + handleToggleRequest(event: Event): void { + const detail = + event instanceof CustomEvent && typeof event.detail === "object" && event.detail !== null + ? (event.detail as DesktopPanelToggleDetail) + : null; + if (detail?.dock === "right" || detail?.dock === "bottom") { + this.dockLayout.setDock(detail.dock, false); + } + if (detail?.open === false) { + this.closePanel(); + return; + } + if (!this.available) { + return; + } + const wasOpen = this.dockLayout.open; + this.dockLayout.setOpen(true); + if (detail?.environmentId) { + void this.connectEnvironment(detail.environmentId, false); + } else if (!wasOpen) { + void this.refreshEnvironments(); + } else if (detail?.open !== true) { + this.closePanel(); + } + } + + private closePanel(): void { + this.returnToPicker(); + this.dockLayout.setOpen(false); + } + + private returnToPicker(): void { + this.disconnectConnection(); + this.state = "picker"; + this.environmentId = null; + this.controlling = false; + this.disconnectedReason = null; + } + + private disconnectConnection(): void { + this.operationId += 1; + const connection = this.connection; + this.connection = null; + connection?.disconnect(); + } + + private async refreshEnvironments(): Promise { + const client = this.client; + if (!client || !this.available) { + return; + } + const operationId = ++this.operationId; + this.loading = true; + this.errorText = null; + try { + const result = await client.request("environments.list", {}); + if (operationId !== this.operationId) { + return; + } + this.environments = result.environments.filter( + (environment) => environment.worker?.desktop === true, + ); + } catch (error) { + if (operationId === this.operationId) { + this.errorText = t("desktop.errors.listFailed", { error: formatUiError(error) }); + } + } finally { + if (operationId === this.operationId) { + this.loading = false; + } + } + } + + private async connectEnvironment( + environmentId: string, + control: boolean, + options: { preserveNotice?: boolean; takeoverRecovery?: boolean } = {}, + ): Promise { + const client = this.client; + if (!client || !this.available) { + return; + } + this.disconnectConnection(); + const operationId = this.operationId; + this.environmentId = environmentId; + this.controlling = control; + this.state = "connecting"; + this.errorText = null; + this.disconnectedReason = null; + if (!options.preserveNotice) { + this.noticeText = null; + } + this.controlTakeoverRecoveryUsed = options.takeoverRecovery === true; + try { + const observed = await client.request("worker.desktop.observe", { + environmentId, + control, + }); + if (operationId !== this.operationId) { + return; + } + await this.updateComplete; + const target = this.shadowRoot?.querySelector(".desktop-surface"); + if (!target) { + throw new Error("Desktop render target is unavailable"); + } + const desktopClient = this.desktopClientFactory(); + const connection = await desktopClient.connect({ + wsUrl: observed.wsPath, + gatewayUrl: client.gatewayUrl, + password: observed.vncPassword, + viewOnly: !observed.control, + target, + onConnect: () => { + if (operationId === this.operationId) { + this.state = "connected"; + } + }, + onDisconnect: (detail) => { + if (operationId === this.operationId) { + this.handleDesktopDisconnect(environmentId, detail.code, detail.reason); + } + }, + onSecurityFailure: (detail) => { + if (operationId === this.operationId) { + this.errorText = t("desktop.errors.securityFailed", { + reason: detail.reason ?? t("desktop.unknownReason"), + }); + } + }, + }); + if (operationId !== this.operationId) { + connection.disconnect(); + return; + } + this.connection = connection; + } catch (error) { + if (operationId === this.operationId) { + this.state = "disconnected"; + this.disconnectedReason = formatUiError(error); + } + } + } + + private handleDesktopDisconnect(environmentId: string, code?: number, reason?: string): void { + this.connection = null; + if ( + code === 4000 && + reason === "control-taken" && + this.controlling && + !this.controlTakeoverRecoveryUsed + ) { + this.noticeText = t("desktop.controlTaken"); + void this.connectEnvironment(environmentId, false, { + preserveNotice: true, + takeoverRecovery: true, + }); + return; + } + this.state = "disconnected"; + this.disconnectedReason = + reason || (code ? t("desktop.closeCode", { code: String(code) }) : null); + } + + private renderHeader() { + const dock = this.dockLayout.dock; + return html` +
+
${t("desktop.title")}
+
+ + + +
+
+ `; + } + + private renderPicker() { + return html` +
+ ${t("desktop.pickerTitle")} + + +
+
+ ${this.loading && this.environments.length === 0 + ? html`
${t("desktop.loading")}
` + : this.environments.length === 0 + ? html`
${t("desktop.empty")}
` + : this.environments.map((environment) => this.renderEnvironment(environment))} +
+ `; + } + + private renderEnvironment(environment: EnvironmentSummary) { + const worker = environment.worker; + return html` +
+
+
${environment.id}
+
+ ${worker?.state ?? environment.status} +
+ ${worker && worker.attachedSessionIds.length > 0 + ? html`
+ ${worker.attachedSessionIds.map( + (sessionId) => html`${sessionId}`, + )} +
` + : nothing} +
+ +
+ `; + } + + private renderConnection() { + return html` +
+ + ${this.controlling ? t("desktop.controlling") : t("desktop.viewOnly")} + + + ${!this.controlling + ? html`` + : nothing} + +
+
+ ${this.state === "connecting" + ? html`
${t("desktop.connecting")}
` + : nothing} + `; + } + + private renderDisconnected() { + return html` +
+
+ ${t("desktop.disconnected", { + reason: this.disconnectedReason ?? t("desktop.unknownReason"), + })} +
+ +
+ `; + } + + override render() { + if (!this.available || !this.dockLayout.open) { + return nothing; + } + const dock = this.dockLayout.dock; + const style = + dock === "bottom" ? `height:${this.dockLayout.height}px` : `width:${this.dockLayout.width}px`; + return html` +
+ ${this.dockLayout.renderResizer("bp", t("desktop.resize"))} ${this.renderHeader()} +
+ ${this.errorText + ? html`` + : this.noticeText + ? html`
${this.noticeText}
` + : nothing} + ${this.state === "picker" + ? this.renderPicker() + : this.state === "disconnected" + ? this.renderDisconnected() + : this.renderConnection()} +
+
+ `; + } +} + +if (!customElements.get("openclaw-desktop-panel")) { + customElements.define("openclaw-desktop-panel", OpenClawDesktopPanel); +} + +declare global { + interface HTMLElementTagNameMap { + "openclaw-desktop-panel": OpenClawDesktopPanel; + } +} diff --git a/ui/src/components/panel-toggle-contract.ts b/ui/src/components/panel-toggle-contract.ts index 817d2745f855..c1e22ee7c86a 100644 --- a/ui/src/components/panel-toggle-contract.ts +++ b/ui/src/components/panel-toggle-contract.ts @@ -2,6 +2,7 @@ import type { UiCommandParams } from "@openclaw/gateway-protocol"; export const TERMINAL_PANEL_TOGGLE_EVENT = "openclaw:terminal-toggle"; export const BROWSER_PANEL_TOGGLE_EVENT = "openclaw:browser-toggle"; +export const DESKTOP_PANEL_TOGGLE_EVENT = "openclaw:desktop-toggle"; export const CUSTODIAN_PANEL_TOGGLE_EVENT = "openclaw:custodian-toggle"; export const UI_COMMAND_EVENT = "openclaw:ui-command"; @@ -24,6 +25,12 @@ export type BrowserPanelToggleDetail = { url?: string; }; +export type DesktopPanelToggleDetail = { + dock?: "bottom" | "right"; + open?: boolean; + environmentId?: string; +}; + export type CustodianPanelToggleDetail = { dock?: "bottom" | "right"; open?: boolean; diff --git a/ui/src/e2e/desktop-panel.e2e.test.ts b/ui/src/e2e/desktop-panel.e2e.test.ts new file mode 100644 index 000000000000..02c0d54fb0e1 --- /dev/null +++ b/ui/src/e2e/desktop-panel.e2e.test.ts @@ -0,0 +1,124 @@ +import { expect, it } from "vitest"; +import { installMockGateway } from "../test-helpers/control-ui-e2e.ts"; +import { createControlUiE2eSuite } from "./control-ui-e2e-suite.test-support.ts"; + +const suite = createControlUiE2eSuite({ + name: "cloud worker desktop panel", + startServerBeforeBrowser: true, + unavailableMessage: (executablePath) => + `Playwright Chromium is not installed or cannot start at ${executablePath}. Run \`pnpm --dir ui exec playwright install --with-deps chromium\`.`, +}); + +async function openPalette(page: import("playwright").Page) { + await page.evaluate(() => { + window.dispatchEvent(new CustomEvent("openclaw:command-palette-open")); + }); + await page.getByRole("combobox", { name: "Search chats and commands…" }).waitFor(); +} + +suite.define(() => { + it("hides the desktop command without the method or operator.admin", async () => { + for (const scenario of [ + { featureMethods: ["environments.list"] }, + { + featureMethods: ["environments.list", "worker.desktop.observe"], + operatorScopes: ["operator.read"], + }, + ]) { + await suite.withPage({ serviceWorkers: "block" }, async ({ page }) => { + await installMockGateway(page, scenario); + await page.goto(`${suite.server.baseUrl}chat`); + await openPalette(page); + expect(await page.getByRole("option", { name: "Desktop", exact: true }).count()).toBe(0); + }); + } + }); + + it("lists a desktop worker and requests view then control observer leases", async () => { + await suite.withPage({ serviceWorkers: "block" }, async ({ page }) => { + const gateway = await installMockGateway(page, { + featureMethods: ["environments.list", "worker.desktop.observe"], + methodResponses: { + "environments.list": { + environments: [ + { + id: "worker-desktop-1", + type: "worker", + status: "available", + worker: { + providerId: "crabbox", + state: "attached", + ageMs: 1_000, + attachedSessionIds: ["agent:main:desktop"], + tunnelStatus: "connected", + desktop: true, + }, + }, + ], + }, + "worker.desktop.observe": { + cases: [ + { + match: { environmentId: "worker-desktop-1", control: false }, + response: { + transport: "rfb", + wsPath: "/worker-desktop/observe?token=view", + expiresAtMs: 60_000, + control: false, + }, + }, + { + match: { environmentId: "worker-desktop-1", control: true }, + response: { + transport: "rfb", + wsPath: "/worker-desktop/observe?token=control", + expiresAtMs: 60_000, + control: true, + }, + }, + ], + }, + }, + }); + + await page.goto(`${suite.server.baseUrl}chat`); + await openPalette(page); + await page.getByRole("option", { name: "Desktop", exact: true }).click(); + + const panel = page.locator("openclaw-desktop-panel"); + await panel.locator("section[aria-label='Desktop']").waitFor(); + await gateway.waitForRequest("environments.list"); + await panel.getByText("worker-desktop-1", { exact: true }).waitFor(); + await panel.getByText("agent:main:desktop", { exact: true }).waitFor(); + await panel.evaluate((element) => { + ( + element as HTMLElement & { + desktopClientFactory: () => { + connect(options: { onConnect?: () => void }): Promise<{ disconnect(): void }>; + }; + } + ).desktopClientFactory = () => ({ + async connect(options) { + queueMicrotask(() => options.onConnect?.()); + return { disconnect() {} }; + }, + }); + }); + + await panel.getByRole("button", { name: "Connect", exact: true }).click(); + const viewRequest = await gateway.waitForRequest("worker.desktop.observe"); + expect(viewRequest.params).toEqual({ environmentId: "worker-desktop-1", control: false }); + + await panel.getByRole("button", { name: "Take control", exact: true }).click(); + await expect + .poll(async () => (await gateway.getRequests("worker.desktop.observe")).length) + .toBe(2); + const observeRequests = await gateway.getRequests("worker.desktop.observe"); + expect(observeRequests[1]?.params).toEqual({ + environmentId: "worker-desktop-1", + control: true, + }); + await panel.getByText("Controlling · view-only for others", { exact: true }).waitFor(); + }); + }); +}); diff --git a/ui/src/i18n/.i18n/raw-copy-baseline.json b/ui/src/i18n/.i18n/raw-copy-baseline.json index 93fe42cefdf4..1217edbf61ce 100644 --- a/ui/src/i18n/.i18n/raw-copy-baseline.json +++ b/ui/src/i18n/.i18n/raw-copy-baseline.json @@ -29,6 +29,13 @@ "path": "ui/src/app/lazy-custom-element.ts", "text": "custodian panel" }, + { + "count": 1, + "kind": "object-property", + "name": "label", + "path": "ui/src/app/lazy-custom-element.ts", + "text": "desktop panel" + }, { "count": 1, "kind": "object-property", diff --git a/ui/src/i18n/locales/en.ts b/ui/src/i18n/locales/en.ts index ab0dcf6710a5..a0c64e4d6451 100644 --- a/ui/src/i18n/locales/en.ts +++ b/ui/src/i18n/locales/en.ts @@ -1906,6 +1906,35 @@ export const en: TranslationMap = { outro: "Please look at the marked area and tell me what you make of it.", }, }, + desktop: { + title: "Desktop", + toggle: "Toggle desktop panel", + hide: "Hide desktop panel", + resize: "Resize desktop panel", + dockBottom: "Dock to bottom", + dockRight: "Dock to right", + pickerTitle: "Cloud worker desktops", + refresh: "Refresh", + refreshing: "Refreshing…", + loading: "Loading worker environments…", + empty: + "No desktop-capable worker environments exist. Enable one with desktop: true in a crabbox cloud-worker profile.", + connect: "Connect", + connecting: "Connecting to desktop…", + viewOnly: "View only", + takeControl: "Take control", + controlling: "Controlling · view-only for others", + disconnect: "Disconnect", + reconnect: "Reconnect", + controlTaken: "Another operator took control", + disconnected: "Desktop disconnected: {reason}", + closeCode: "connection closed with code {code}", + unknownReason: "unknown reason", + errors: { + listFailed: "Could not load worker environments: {error}", + securityFailed: "Desktop security negotiation failed: {reason}", + }, + }, routeTitles: { modelProviders: "Models", notifications: "Notifications", @@ -2769,6 +2798,11 @@ export const en: TranslationMap = { description: "Record content-free metadata for direct conversations in the audit ledger. Message content is never stored.", }, + workerDesktop: { + title: "Cloud Worker Desktop", + description: + "Watch and control desktop-capable cloud worker environments live from a Desktop panel; requires crabbox profiles with desktop: true.", + }, }, aboutPage: { productName: "OpenClaw", @@ -3616,6 +3650,7 @@ export const en: TranslationMap = { plugins: "Plugins", settings: "Settings", agents: "Agents", + desktop: "Desktop", }, descriptions: { verboseMode: "Toggle verbose mode.", diff --git a/ui/src/pages/labs/labs-page.test.ts b/ui/src/pages/labs/labs-page.test.ts index c6cc6ed8dd13..0e175832af1a 100644 --- a/ui/src/pages/labs/labs-page.test.ts +++ b/ui/src/pages/labs/labs-page.test.ts @@ -104,6 +104,7 @@ describe("LabsPage", () => { expect(page.querySelectorAll(".settings-row")).toHaveLength(LAB_FEATURES.length); expect(page.textContent).toContain("Code Mode"); expect(page.textContent).toContain("Swarm"); + expect(page.textContent).toContain("Cloud Worker Desktop"); expect(codeModeToggle(page).checked).toBe(true); const docs = [...page.querySelectorAll(".settings-row__desc a")]; @@ -198,6 +199,13 @@ describe("LabsPage", () => { expectedPatch: { logging: { audit: { messages: "direct" } } }, note: "labs: update auditMessages", }, + { + label: "Cloud Worker Desktop", + index: 6, + sourceConfig: { cloudWorkers: { desktop: false } }, + expectedPatch: { cloudWorkers: { desktop: true } }, + note: "labs: update workerDesktop", + }, ])("writes the on value at the registered config path when enabling $label", async (testCase) => { const { page, runtimeConfig } = await mountPage(testCase.sourceConfig); const toggle = labToggle(page, testCase.index, testCase.label); @@ -247,13 +255,18 @@ describe("LabsPage", () => { }); }); - it("marks only the startup-scoped entry as needing a restart", async () => { + it("marks startup-scoped entries as needing a restart", async () => { const { page } = await mountPage({}); const rows = [...page.querySelectorAll(".settings-row")]; const restartRows = rows.filter((row) => row.textContent?.includes("restart")); - expect(restartRows).toHaveLength(1); - expect(restartRows[0]?.textContent).toContain("Message audit metadata"); + expect(restartRows).toHaveLength(2); + expect(restartRows.map((row) => row.textContent)).toEqual( + expect.arrayContaining([ + expect.stringContaining("Message audit metadata"), + expect.stringContaining("Cloud Worker Desktop"), + ]), + ); }); it("shows default provenance and reset actions only for overrides", async () => { diff --git a/ui/src/pages/labs/labs-registry.ts b/ui/src/pages/labs/labs-registry.ts index 0d4e0667091a..11c695fb09a6 100644 --- a/ui/src/pages/labs/labs-registry.ts +++ b/ui/src/pages/labs/labs-registry.ts @@ -193,6 +193,21 @@ export const LAB_FEATURES = [ // the recorder, so this outlives the reload plan's `logging: none` rule. restartHint: () => t("labsPage.restartRequired"), }, + { + id: "workerDesktop", + title: () => t("labsPage.workerDesktop.title"), + description: () => t("labsPage.workerDesktop.description"), + docsUrl: "https://docs.openclaw.ai/gateway/cloud-workers#desktop-interactive", + configPath: ["cloudWorkers", "desktop"], + onValue: true, + offValue: false, + activeValues: [true], + readEnabled: null, + enableAlso: null, + resetScope: "gate", + // Method advertisement is resolved at Gateway startup, so the panel appears after restart. + restartHint: () => t("labsPage.restartRequired"), + }, ] as const satisfies readonly LabFeature[]; function recordAtPath(config: Record, path: readonly string[]): unknown { diff --git a/ui/src/types/novnc.d.ts b/ui/src/types/novnc.d.ts new file mode 100644 index 000000000000..ac4aa2a4543d --- /dev/null +++ b/ui/src/types/novnc.d.ts @@ -0,0 +1,13 @@ +declare module "@novnc/novnc" { + export default class RFB extends EventTarget { + constructor( + target: HTMLElement, + channel: string | WebSocket, + options?: { credentials?: { password: string } }, + ); + + scaleViewport: boolean; + viewOnly: boolean; + disconnect(): void; + } +}