From f695be341c52aa0248a57e115bc9a1309b2e4de2 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 21 Jul 2026 05:58:04 -0700 Subject: [PATCH] feat(cua-computer): add experimental Windows/Linux computer-use fulfiller (#112267) * feat(cua-computer): add experimental Windows/Linux computer-use fulfiller Bundled plugin that fulfills the capability-based computer.act + screen.snapshot node contract on Windows and Linux by supervising a pinned cua-driver 0.10.x daemon over MCP stdio. macOS keeps the Peekaboo fulfiller; this plugin is disabled by default and never available on darwin. Grounded in cua-driver 0.10.0 source (tool schemas, refusal codes, coordinate spaces, session/daemon lifecycle). Notable safety and correctness properties: - Deny-by-default env allowlist so OpenClaw secrets (provider/channel tokens, CUA_API_KEY) never reach the separately installed daemon; telemetry and update checks forced off. - Version-gated handshake (exact-minor pin + capability/schema version), time-bounded so a corrected driver recovers without a node restart. - Robust daemon supervision: full readiness-budget polling, startup-race tolerance, signal-death and spawn-error recovery, shared-daemon lifecycle (never killed on dispose). - Frame authorization preserved within upstream limits (generation + full live geometry; capture refused when screen and screenshot geometry diverge). - Action mapping refuses inputs cua-driver cannot faithfully deliver: layout-shifted keys, modifier-held drag/scroll, Linux modifier clicks, hold_key/mouse down-up, non-positive scroll; drag duration clamped. * fix(cua-computer): satisfy lint, test-types, dead-code, and docs-map gates --- .github/labeler.yml | 4 + docs/docs_map.md | 9 + docs/nodes/computer-use.md | 30 +- docs/plugins/plugin-inventory.md | 6 +- docs/plugins/reference.md | 2 +- docs/plugins/reference/cua-computer.md | 19 + docs/plugins/reference/mxc.md | 4 +- extensions/cua-computer/index.test.ts | 21 + extensions/cua-computer/index.ts | 34 + extensions/cua-computer/openclaw.plugin.json | 18 + extensions/cua-computer/package.json | 19 + extensions/cua-computer/src/actions.test.ts | 81 +++ extensions/cua-computer/src/actions.ts | 176 +++++ extensions/cua-computer/src/commands.test.ts | 602 ++++++++++++++++++ extensions/cua-computer/src/commands.ts | 462 ++++++++++++++ .../cua-computer/src/driver-client.test.ts | 480 ++++++++++++++ extensions/cua-computer/src/driver-client.ts | 514 +++++++++++++++ extensions/cua-computer/src/frame.ts | 114 ++++ pnpm-lock.yaml | 16 + 19 files changed, 2601 insertions(+), 10 deletions(-) create mode 100644 docs/plugins/reference/cua-computer.md create mode 100644 extensions/cua-computer/index.test.ts create mode 100644 extensions/cua-computer/index.ts create mode 100644 extensions/cua-computer/openclaw.plugin.json create mode 100644 extensions/cua-computer/package.json create mode 100644 extensions/cua-computer/src/actions.test.ts create mode 100644 extensions/cua-computer/src/actions.ts create mode 100644 extensions/cua-computer/src/commands.test.ts create mode 100644 extensions/cua-computer/src/commands.ts create mode 100644 extensions/cua-computer/src/driver-client.test.ts create mode 100644 extensions/cua-computer/src/driver-client.ts create mode 100644 extensions/cua-computer/src/frame.ts diff --git a/.github/labeler.yml b/.github/labeler.yml index c886df098be8..5ee66afd3681 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -17,6 +17,10 @@ - "docs/nodes/camera.md" - "docs/nodes/location-command.md" - "docs/platforms/linux.md" +"plugin: cua-computer": + - changed-files: + - any-glob-to-any-file: + - "extensions/cua-computer/**" "plugin: pixverse": - changed-files: - any-glob-to-any-file: diff --git a/docs/docs_map.md b/docs/docs_map.md index dda783bf1221..d68b7e9397c8 100644 --- a/docs/docs_map.md +++ b/docs/docs_map.md @@ -4943,6 +4943,7 @@ Do not edit it by hand; run `pnpm docs:map:gen`. - Headings: - H2: Requirements - H2: The computer agent tool + - H2: Windows and Linux (experimental, via cua-driver) - H2: The computer.act node command - H2: Enable and arm - H2: Safety @@ -6382,6 +6383,14 @@ Do not edit it by hand; run `pnpm docs:map:gen`. - H2: Surface - H2: Configure +## plugins/reference/cua-computer.md + +- Route: /plugins/reference/cua-computer +- Headings: + - H1: Cua Computer plugin + - H2: Distribution + - H2: Surface + ## plugins/reference/deepgram.md - Route: /plugins/reference/deepgram diff --git a/docs/nodes/computer-use.md b/docs/nodes/computer-use.md index c00bfdaa66e1..d5492d381e8c 100644 --- a/docs/nodes/computer-use.md +++ b/docs/nodes/computer-use.md @@ -9,13 +9,14 @@ title: "Computer use" Computer use lets the gateway agent see and control a capable paired desktop. Eligibility is capability-based: the connected node must advertise both `computer.act` and `screen.snapshot`, whose result must include a `displayFrameId`. The tool captures a screenshot as its reference frame, then drives the pointer and keyboard through the dangerous `computer.act` command. The action set follows the core Anthropic computer-use actions; optional `computer_20251124` zoom is not exposed. A vision-capable model drives it through the built-in `computer` agent tool. -The agent emits one uniform command, `computer.act`; it cannot tell how a node fulfills it. The bundled macOS app is currently the only shipped fulfiller and handles the command in-process with embedded Peekaboo services plus narrow CoreGraphics primitives (correct TCC permissions, no extra process). Windows and Linux desktop nodes may declare both `computer.act` and `screen.snapshot` under the same pairing and arming policy, but their platform apps do not fulfill desktop control yet. Future fulfillers can implement this command pair without changing the agent-facing contract. +The agent emits one uniform command, `computer.act`; it cannot tell how a node fulfills it. The bundled macOS app handles the command in-process with embedded Peekaboo services plus narrow CoreGraphics primitives (correct TCC permissions, no extra process). Windows and Linux can use the optional, experimental `cua-computer` plugin with a separately installed `cua-driver` binary. Both fulfillers use the same pairing and arming policy. ## Requirements -- A paired, connected node advertising both `computer.act` and `screen.snapshot`, with `screen.snapshot` returning `displayFrameId`. Today, the bundled macOS app is the only shipped fulfiller. +- A paired, connected node advertising both `computer.act` and `screen.snapshot`, with `screen.snapshot` returning `displayFrameId`. - **macOS fulfiller:** app setting **Allow Computer Control** enabled (default: off). - **macOS fulfiller:** **Accessibility** permission granted to OpenClaw (for pointer/keyboard injection) and **Screen Recording** permission (for `screen.snapshot`). +- **Windows/Linux fulfiller:** bundled `cua-computer` plugin enabled and a compatible `cua-driver` 0.10.x executable installed. - The `computer.act` command armed on the gateway (it is dangerous and disarmed by default). - A vision-capable agent model. - Tool policy that exposes `computer`. The default `coding` profile does not. Add `computer` to `tools.alsoAllow`; sandboxed agents also need it in `tools.sandbox.tools.alsoAllow`. @@ -34,18 +35,37 @@ Modifier keys ride the `text` field on click and scroll actions (`shift`, `ctrl` Screenshots are kept **model-only**: they are never auto-delivered to the chat channel. Treat all on-screen content as untrusted input; the tool warns the model not to follow on-screen instructions that conflict with the user's request. +## Windows and Linux (experimental, via cua-driver) + +The bundled `cua-computer` plugin provides an experimental fulfiller for Windows and Linux node hosts. It is disabled by default and requires the prerelease 0.10.x driver contract: + +1. Install a `cua-driver` 0.10.x binary from the [upstream releases](https://github.com/trycua/cua/releases) and make it available on `PATH`. To use another executable location, set `plugins.entries.cua-computer.config.driverPath`. +2. Enable the plugin: + + ```bash + openclaw plugins enable cua-computer + ``` + +3. Start `openclaw node run` from the interactive desktop session. The plugin starts the local driver daemon lazily when the first capture or action arrives. + +This fulfiller currently controls only the primary display. X11/XWayland is the first-line Linux path. Native Wayland remains an upstream opt-in: set `CUA_DRIVER_RS_ENABLE_WAYLAND` yourself before starting the node; OpenClaw never sets it automatically. KDE/KWin is unsupported by the upstream native-Wayland input path. `hold_key`, `left_mouse_down`, and `left_mouse_up` are unavailable because cua-driver 0.10.x has no cross-platform desktop-scope hold contract. Modifier-held scrolling and dragging are unavailable on both platforms, and modifier-held clicks are unavailable on Linux. The `key` action accepts named keys, letters, and modifier combos (for example `cmd+c` or `Return`); digit and punctuation keys are rejected because the driver drops their layout-dependent shift state, so send that text through the `type` action instead. Text typing cannot be cancelled partway through a `type_text` driver call. + +Because cua-driver reports no stable display identity, frame authorization binds to the driver connection plus the live primary-display geometry. A daemon or session reconnect invalidates outstanding frames, but a same-geometry primary-display substitution that keeps the connection open cannot be detected; prefer a stable single-display session for this fulfiller. + +OpenClaw disables cua-driver telemetry and update checks for the `mcp` and `serve` processes it manages. It does not download or update the driver binary. + ## The `computer.act` node command `computer.act` is the single node command the tool routes input through (`node.invoke` with `command: "computer.act"`). It is: - **Dangerous by default**: listed in the built-in dangerous node commands and excluded from the runtime allowlist until explicitly armed. macOS, Windows, and Linux desktop nodes may still declare it at pairing so the surface is approved once. -- **Capability-based**: the tool requires a connected node to advertise both `computer.act` and `screen.snapshot`. The bundled macOS app is currently the only shipped fulfiller; Windows/Linux platform-app fulfillers are still to come. +- **Capability-based**: the tool requires a connected node to advertise both `computer.act` and `screen.snapshot`. The bundled macOS app and the opt-in experimental `cua-computer` plugin fulfill the same command pair. Reads reuse `screen.snapshot`; there is no second capture path. See [Camera and screen nodes](/nodes/camera) for the shared capture command. ## Enable and arm -1. For the current macOS fulfiller, enable **Settings → Allow Computer Control**. Then open **Settings → Permissions** and grant **Accessibility** and **Screen Recording** in macOS System Settings. +1. Enable the platform fulfiller: on macOS, enable **Settings → Allow Computer Control**, then grant **Accessibility** and **Screen Recording** under **Settings → Permissions**; on Windows/Linux, follow the experimental `cua-computer` setup above. 2. Approve the pairing update on the gateway (a new command forces re-pairing). 3. Expose the tool to the vision-capable agent. For the default `coding` profile: @@ -82,7 +102,7 @@ enable invocation by itself. ## Safety - Before authorization, every layer (tool policy, gateway command policy, node-app setting, and platform permissions) must agree. For the current macOS fulfiller, that includes **Allow Computer Control**, Accessibility, and Screen Recording. Once armed, actions execute without a per-action confirmation until expiry or `/phone disarm`. -- Text input is posted one grapheme at a time. Cancellation, disconnect, pause, disable, or endpoint replacement stops it before the next grapheme instead of letting the stale remainder continue. +- The macOS fulfiller posts text one grapheme at a time, so cancellation, disconnect, pause, disable, or endpoint replacement stops it before the next grapheme. The experimental cua-driver fulfiller cannot cancel a `type_text` call mid-typing. - Screenshots are model-only and never auto-sent to chat (issue [#44759](https://github.com/openclaw/openclaw/issues/44759)). - Treat screen content as untrusted; it can carry prompt injection. diff --git a/docs/plugins/plugin-inventory.md b/docs/plugins/plugin-inventory.md index a52b5c909b32..b2978c57a7a0 100644 --- a/docs/plugins/plugin-inventory.md +++ b/docs/plugins/plugin-inventory.md @@ -51,7 +51,7 @@ Each entry lists the package, distribution route, and description. ## Core npm package -69 plugins +70 plugins - **[admin-http-rpc](/plugins/reference/admin-http-rpc)** (`@openclaw/admin-http-rpc`) - included in OpenClaw. OpenClaw admin HTTP RPC endpoint. @@ -79,6 +79,8 @@ Each entry lists the package, distribution route, and description. - **[crabbox](/plugins/reference/crabbox)** (`@openclaw/crabbox-provider`) - included in OpenClaw. Cloud worker provider backed by the Crabbox CLI. +- **[cua-computer](/plugins/reference/cua-computer)** (`@openclaw/cua-computer`) - included in OpenClaw. Experimental cua-driver computer control for Windows and Linux node hosts. + - **[deepgram](/plugins/reference/deepgram)** (`@openclaw/deepgram-provider`) - included in OpenClaw. Adds media understanding provider support. Adds realtime transcription provider support. - **[document-extract](/plugins/reference/document-extract)** (`@openclaw/document-extract-plugin`) - included in OpenClaw. Extract text and fallback page images from local document attachments. @@ -281,7 +283,7 @@ Each entry lists the package, distribution route, and description. - **[msteams](/plugins/reference/msteams)** (`@openclaw/msteams`) - npm; ClawHub. OpenClaw Microsoft Teams channel plugin for bot conversations. -- **[mxc](/plugins/reference/mxc)** (`@openclaw/mxc-sandbox`) - npm; ClawHub. OS-level sandboxed tool execution via MXC for MXC-capable Windows hosts: runs commands in ProcessContainer (Windows) with configured MXC policy files. +- **[mxc](/plugins/reference/mxc)** (`@openclaw/mxc-sandbox`) - npm; ClawHub. OS-level sandboxed tool execution via MXC: runs commands in a Windows ProcessContainer with configured MXC policy files. - **[nextcloud-talk](/plugins/reference/nextcloud-talk)** (`@openclaw/nextcloud-talk`) - npm; ClawHub. OpenClaw Nextcloud Talk channel plugin for conversations. diff --git a/docs/plugins/reference.md b/docs/plugins/reference.md index 11258cad202a..a5c6e8709e28 100644 --- a/docs/plugins/reference.md +++ b/docs/plugins/reference.md @@ -15,5 +15,5 @@ This page is generated from `extensions/*/package.json` and pnpm plugins:inventory:gen ``` -Use [Plugin inventory](/plugins/plugin-inventory) to browse all 142 +Use [Plugin inventory](/plugins/plugin-inventory) to browse all 143 generated plugin reference pages by distribution, package, and description. diff --git a/docs/plugins/reference/cua-computer.md b/docs/plugins/reference/cua-computer.md new file mode 100644 index 000000000000..89bf157669dc --- /dev/null +++ b/docs/plugins/reference/cua-computer.md @@ -0,0 +1,19 @@ +--- +summary: "Experimental cua-driver computer control for Windows and Linux node hosts." +read_when: + - You are installing, configuring, or auditing the cua-computer plugin +title: "Cua Computer plugin" +--- + +# Cua Computer plugin + +Experimental cua-driver computer control for Windows and Linux node hosts. + +## Distribution + +- Package: `@openclaw/cua-computer` +- Install route: included in OpenClaw + +## Surface + +plugin diff --git a/docs/plugins/reference/mxc.md b/docs/plugins/reference/mxc.md index e4a3e98325ea..dfe5f231cdc2 100644 --- a/docs/plugins/reference/mxc.md +++ b/docs/plugins/reference/mxc.md @@ -1,5 +1,5 @@ --- -summary: "OS-level sandboxed tool execution via MXC for MXC-capable Windows hosts: runs commands in ProcessContainer (Windows) with configured MXC policy files." +summary: "OS-level sandboxed tool execution via MXC: runs commands in a Windows ProcessContainer with configured MXC policy files." read_when: - You are installing, configuring, or auditing the mxc plugin title: "Mxc plugin" @@ -7,7 +7,7 @@ title: "Mxc plugin" # Mxc plugin -OS-level sandboxed tool execution via MXC for MXC-capable Windows hosts: runs commands in ProcessContainer (Windows) with configured MXC policy files. +OS-level sandboxed tool execution via MXC: runs commands in a Windows ProcessContainer with configured MXC policy files. ## Distribution diff --git a/extensions/cua-computer/index.test.ts b/extensions/cua-computer/index.test.ts new file mode 100644 index 000000000000..b422687a3386 --- /dev/null +++ b/extensions/cua-computer/index.test.ts @@ -0,0 +1,21 @@ +import type { + OpenClawPluginApi, + OpenClawPluginNodeHostCommand, +} from "openclaw/plugin-sdk/plugin-entry"; +import { describe, expect, it } from "vitest"; +import plugin from "./index.js"; + +describe("cua-computer plugin registration", () => { + it("registers the screen and dangerous computer node-host commands", () => { + const commands: OpenClawPluginNodeHostCommand[] = []; + plugin.register({ + pluginConfig: { driverPath: "cua-driver" }, + registerNodeHostCommand: (command: OpenClawPluginNodeHostCommand) => commands.push(command), + } as unknown as OpenClawPluginApi); + + expect(commands.map(({ command, cap, dangerous }) => ({ command, cap, dangerous }))).toEqual([ + { command: "screen.snapshot", cap: "screen", dangerous: false }, + { command: "computer.act", cap: "computer", dangerous: true }, + ]); + }); +}); diff --git a/extensions/cua-computer/index.ts b/extensions/cua-computer/index.ts new file mode 100644 index 000000000000..29dcd47f3870 --- /dev/null +++ b/extensions/cua-computer/index.ts @@ -0,0 +1,34 @@ +import { buildPluginConfigSchema, definePluginEntry } from "openclaw/plugin-sdk/plugin-entry"; +import { z } from "zod"; +import { createCuaComputerCommands } from "./src/commands.js"; + +const CuaComputerConfigSchema = z.strictObject({ + driverPath: z.string().trim().min(1).optional(), +}); + +const configSchema = buildPluginConfigSchema(CuaComputerConfigSchema, { + uiHints: { + driverPath: { + label: "cua-driver path", + help: "Absolute path or executable name resolved through PATH. Defaults to cua-driver.", + }, + }, +}); + +export default definePluginEntry({ + id: "cua-computer", + name: "CUA Computer", + description: "Experimental cua-driver computer control for Windows and Linux node hosts.", + configSchema, + register(api) { + const parsed = CuaComputerConfigSchema.safeParse(api.pluginConfig ?? {}); + if (!parsed.success) { + throw new Error( + `Invalid cua-computer plugin config: ${parsed.error.issues[0]?.message ?? "invalid config"}`, + ); + } + for (const command of createCuaComputerCommands({ driverPath: parsed.data.driverPath })) { + api.registerNodeHostCommand(command); + } + }, +}); diff --git a/extensions/cua-computer/openclaw.plugin.json b/extensions/cua-computer/openclaw.plugin.json new file mode 100644 index 000000000000..a1b28bae4cfe --- /dev/null +++ b/extensions/cua-computer/openclaw.plugin.json @@ -0,0 +1,18 @@ +{ + "id": "cua-computer", + "activation": { + "onStartup": true + }, + "enabledByDefault": false, + "name": "CUA Computer", + "description": "Experimental cua-driver computer control for Windows and Linux node hosts.", + "configSchema": { + "type": "object", + "additionalProperties": false, + "properties": { + "driverPath": { + "type": "string" + } + } + } +} diff --git a/extensions/cua-computer/package.json b/extensions/cua-computer/package.json new file mode 100644 index 000000000000..20b597025be7 --- /dev/null +++ b/extensions/cua-computer/package.json @@ -0,0 +1,19 @@ +{ + "name": "@openclaw/cua-computer", + "version": "2026.7.2", + "description": "Experimental cua-driver computer control for Windows and Linux node hosts", + "type": "module", + "dependencies": { + "@modelcontextprotocol/sdk": "1.29.0", + "rastermill": "0.3.1", + "zod": "4.4.3" + }, + "devDependencies": { + "@openclaw/plugin-sdk": "workspace:*" + }, + "openclaw": { + "extensions": [ + "./index.ts" + ] + } +} diff --git a/extensions/cua-computer/src/actions.test.ts b/extensions/cua-computer/src/actions.test.ts new file mode 100644 index 000000000000..60ce327a6d9d --- /dev/null +++ b/extensions/cua-computer/src/actions.test.ts @@ -0,0 +1,81 @@ +import { describe, expect, it } from "vitest"; +import { normalizeModifiers, parseKeyChord, scalePoint } from "./actions.js"; + +// normalizeKey is internal; exercise it through parseKeyChord's final segment. +const normalizeKey = (input: string) => parseKeyChord(input).key; + +describe("cua-computer key normalization", () => { + it.each([ + ["cmd+shift", ["meta", "shift"]], + ["Super+Control+Option", ["meta", "ctrl", "alt"]], + ["win+mod1", ["meta", "alt"]], + ])("normalizes modifier aliases in %s", (input, expected) => { + expect(normalizeModifiers(input)).toEqual(expected); + }); + + it.each([ + ["Return", "enter"], + ["Esc", "escape"], + ["PgDn", "pagedown"], + ["Home", "home"], + ["F12", "f12"], + ["z", "z"], + ["Z", "z"], + ["c", "c"], + ])("normalizes key %s", (input, expected) => { + expect(normalizeKey(input)).toBe(expected); + }); + + it.each(["minus", "slash", "equal", "period", "comma", "semicolon"])( + "rejects punctuation-alias key %s toward the type action", + (input) => { + expect(() => normalizeKey(input)).toThrow("COMPUTER_UNSUPPORTED_KEY"); + }, + ); + + it("keeps letter keys usable in shortcut chords", () => { + expect(parseKeyChord("cmd+c")).toEqual({ key: "c", modifiers: ["meta"] }); + }); + + // Digits and punctuation are shifted on some keyboard layouts, and cua-driver + // drops that shift state, so they must be rejected toward the type action + // rather than silently degraded. + it.each(["1", "+", "*", ":", "_", "(", ".", "?", "é"])( + "rejects layout-shifted key %s toward the type action", + (input) => { + expect(() => normalizeKey(input)).toThrow("COMPUTER_UNSUPPORTED_KEY"); + }, + ); + + it("splits the last chord segment into the key", () => { + expect(parseKeyChord("cmd+ctrl+Return")).toEqual({ + key: "enter", + modifiers: ["meta", "ctrl"], + }); + }); + + it.each(["hyper", "ctrl+hyper"])("rejects unknown vocabulary in %s", (input) => { + const operation = input.includes("+") + ? () => parseKeyChord(input) + : () => normalizeModifiers(input); + expect(operation).toThrow("COMPUTER_UNSUPPORTED_KEY"); + }); + + it("keeps rounded coordinates inside the native primary-display bounds", () => { + expect( + scalePoint( + { + id: "frame", + nativeWidth: 3840, + nativeHeight: 2160, + deliveredWidth: 1920, + deliveredHeight: 1080, + geometry: { width: 3840, height: 2160, scaleFactor: 1 }, + }, + 1919.9, + 1079.9, + "click", + ), + ).toEqual({ x: 3839, y: 2159 }); + }); +}); diff --git a/extensions/cua-computer/src/actions.ts b/extensions/cua-computer/src/actions.ts new file mode 100644 index 000000000000..770d610064d1 --- /dev/null +++ b/extensions/cua-computer/src/actions.ts @@ -0,0 +1,176 @@ +import { z } from "zod"; +import type { CuaLastFrame } from "./frame.js"; + +const COMPUTER_ACTIONS = [ + "left_click", + "right_click", + "middle_click", + "double_click", + "triple_click", + "mouse_move", + "left_click_drag", + "left_mouse_down", + "left_mouse_up", + "scroll", + "type", + "key", + "hold_key", +] as const; + +export const ComputerActParamsSchema = z.strictObject({ + action: z.enum(COMPUTER_ACTIONS), + displayFrameId: z.string().optional(), + x: z.number().finite().nonnegative().optional(), + y: z.number().finite().nonnegative().optional(), + fromX: z.number().finite().nonnegative().optional(), + fromY: z.number().finite().nonnegative().optional(), + text: z.string().optional(), + keys: z.string().optional(), + modifiers: z.string().optional(), + scrollDirection: z.enum(["up", "down", "left", "right"]).optional(), + scrollAmount: z.number().int().positive().optional(), + durationMs: z.number().int().nonnegative().optional(), + screenIndex: z.number().int().nonnegative().optional(), + refWidth: z.number().int().positive().optional(), +}); + +export type ComputerActParams = z.infer; + +const MODIFIER_ALIASES = new Map([ + ["ctrl", "ctrl"], + ["control", "ctrl"], + ["shift", "shift"], + ["alt", "alt"], + ["menu", "alt"], + ["option", "alt"], + ["mod1", "alt"], + ["cmd", "meta"], + ["command", "meta"], + ["meta", "meta"], + ["super", "meta"], + ["win", "meta"], + ["windows", "meta"], + ["mod4", "meta"], +]); + +const KEY_ALIASES = new Map([ + ["return", "enter"], + ["enter", "enter"], + ["tab", "tab"], + ["escape", "escape"], + ["esc", "escape"], + ["space", "space"], + ["backspace", "backspace"], + ["delete", "delete"], + ["del", "delete"], + ["insert", "insert"], + ["ins", "insert"], + ["home", "home"], + ["end", "end"], + ["pageup", "pageup"], + ["pgup", "pageup"], + ["pagedown", "pagedown"], + ["pgdn", "pagedown"], + ["up", "up"], + ["down", "down"], + ["left", "left"], + ["right", "right"], + ["capslock", "capslock"], + ["numlock", "numlock"], + // Punctuation aliases are intentionally absent: they resolve to characters + // whose shift/AltGr state is layout-dependent and dropped by cua-driver, the + // same reason single punctuation chars are rejected below. Route them to the + // `type` action instead. +]); + +for (let index = 1; index <= 12; index += 1) { + KEY_ALIASES.set(`f${index}`, `f${index}`); +} + +function unsupportedKey(message: string): Error { + return new Error(`COMPUTER_UNSUPPORTED_KEY: ${message}`); +} + +export function normalizeModifiers(value: string | undefined): string[] { + if (!value?.trim()) { + return []; + } + return value.split("+").map((entry) => { + const raw = entry.trim(); + const normalized = MODIFIER_ALIASES.get(raw.toLowerCase()); + if (!normalized) { + throw unsupportedKey(`unknown modifier ${JSON.stringify(raw)}`); + } + return normalized; + }); +} + +function normalizeKey(value: string): string { + const raw = value.trim(); + if (!raw) { + throw unsupportedKey("key chord contains an empty key"); + } + const lowered = raw.toLowerCase(); + const modifier = MODIFIER_ALIASES.get(lowered); + if (modifier) { + return modifier; + } + const named = KEY_ALIASES.get(lowered); + if (named) { + return named; + } + // cua-driver 0.10 resolves single characters through VkKeyScanW/keysym lookups + // and keeps only the base virtual key, dropping the shift/AltGr state the + // active layout needs (keyboard.rs key_name_to_vk). ASCII letters are unshifted + // in every Latin layout, so they stay valid chord keys (e.g. ctrl+c). Digits + // and punctuation are shifted on some layouts (AZERTY digits, US symbols), so + // they are rejected toward the `type` action rather than mis-sent. + if (/^[a-z]$/i.test(raw)) { + return lowered; + } + if (raw.length === 1) { + throw unsupportedKey( + `single-character key ${JSON.stringify(raw)} loses layout shift state in cua-driver; use the type action instead`, + ); + } + throw unsupportedKey(`unknown key ${JSON.stringify(raw)}`); +} + +export function parseKeyChord(value: string | undefined): { key: string; modifiers: string[] } { + const segments = value?.split("+").map((entry) => entry.trim()) ?? []; + const rawKey = segments.pop(); + if (!rawKey) { + throw unsupportedKey("key chord is empty"); + } + const modifiers = segments.map((entry) => { + const normalized = MODIFIER_ALIASES.get(entry.toLowerCase()); + if (!normalized) { + throw unsupportedKey(`unknown modifier ${JSON.stringify(entry)}`); + } + return normalized; + }); + return { key: normalizeKey(rawKey), modifiers }; +} + +export function scalePoint( + frame: CuaLastFrame, + x: number | undefined, + y: number | undefined, + label: string, +): { x: number; y: number } { + if (x === undefined || y === undefined) { + throw new Error(`COMPUTER_INVALID_REQUEST: ${label} coordinates are required`); + } + if (x >= frame.deliveredWidth || y >= frame.deliveredHeight) { + throw new Error( + `COMPUTER_INVALID_REQUEST: ${label} coordinates are outside the captured primary-display frame`, + ); + } + return { + x: Math.min(frame.nativeWidth - 1, Math.round((x * frame.nativeWidth) / frame.deliveredWidth)), + y: Math.min( + frame.nativeHeight - 1, + Math.round((y * frame.nativeHeight) / frame.deliveredHeight), + ), + }; +} diff --git a/extensions/cua-computer/src/commands.test.ts b/extensions/cua-computer/src/commands.test.ts new file mode 100644 index 000000000000..a64f17c44bea --- /dev/null +++ b/extensions/cua-computer/src/commands.test.ts @@ -0,0 +1,602 @@ +import { describe, expect, it, vi } from "vitest"; +import { createCuaComputerCommands } from "./commands.js"; +import type { CuaDriver, CuaToolResult } from "./driver-client.js"; + +type ToolCall = { name: string; args: Record }; + +function desktopResult(overrides: Record = {}): CuaToolResult { + return { + content: [ + { type: "image", data: Buffer.from("native-png").toString("base64"), mimeType: "image/png" }, + { type: "text", text: "desktop" }, + ], + structuredContent: { + platform: "linux", + display: "primary", + screenshot_width: 3840, + screenshot_height: 2160, + screen_width: 3840, + screen_height: 2160, + scale_factor: 1, + ...overrides, + }, + }; +} + +function screenSizeResult(overrides: Record = {}): CuaToolResult { + return { + content: [{ type: "text", text: "size" }], + structuredContent: { width: 3840, height: 2160, scale_factor: 1, ...overrides }, + }; +} + +function createDriver( + options: { + desktop?: () => CuaToolResult; + callTool?: (name: string, args: Record) => Promise; + available?: boolean; + generation?: () => number; + } = {}, +) { + const calls: ToolCall[] = []; + const driver: CuaDriver = { + get generation() { + return options.generation?.() ?? 1; + }, + isAvailable: () => options.available ?? true, + resetAvailabilityCache: vi.fn(), + callTool: async (name, args) => { + calls.push({ name, args }); + if (options.callTool) { + return await options.callTool(name, args); + } + if (name === "get_desktop_state") { + return options.desktop?.() ?? desktopResult(); + } + if (name === "get_screen_size") { + return screenSizeResult(); + } + return { content: [{ type: "text", text: "ok" }] }; + }, + dispose: vi.fn(async () => {}), + }; + return { driver, calls }; +} + +function createProcessor() { + const encode = vi.fn( + async ( + _input: Buffer, + options: { format: "jpeg" | "png"; quality?: number; resize?: { width: number } }, + ) => ({ + data: Buffer.from(`${options.format}-encoded`), + width: options.resize?.width ?? 3840, + height: options.resize ? Math.round((2160 * options.resize.width) / 3840) : 2160, + }), + ); + return { processor: { encode }, encode }; +} + +function commandSet( + driver: CuaDriver, + imageProcessor = createProcessor().processor, + platform: NodeJS.Platform = "linux", +) { + const commands = createCuaComputerCommands({ platform, driver, imageProcessor }); + const snapshot = commands.find((command) => command.command === "screen.snapshot"); + const act = commands.find((command) => command.command === "computer.act"); + if (!snapshot || !act) { + throw new Error("commands missing"); + } + return { snapshot, act }; +} + +async function issueFrameFor(driver: CuaDriver, platform: NodeJS.Platform = "linux") { + const { snapshot, act } = commandSet(driver, createProcessor().processor, platform); + const payload = JSON.parse( + await snapshot.handle(JSON.stringify({ maxWidth: 1920, format: "jpeg" })), + ) as { displayFrameId: string; width: number }; + return { act, frameId: payload.displayFrameId, refWidth: payload.width }; +} + +describe("cua-computer screen.snapshot", () => { + it("scales native screenshots to maxWidth and returns delivered dimensions", async () => { + const { driver } = createDriver(); + const { processor, encode } = createProcessor(); + const { snapshot } = commandSet(driver, processor); + + const payload = JSON.parse( + await snapshot.handle(JSON.stringify({ maxWidth: 1456, quality: 0.61, format: "jpeg" })), + ) as Record; + + expect(payload).toMatchObject({ format: "jpeg", screenIndex: 0, width: 1456, height: 819 }); + expect(payload.displayFrameId).toMatch(/^cua:v1:[a-f0-9]{64}$/); + expect(encode).toHaveBeenCalledWith( + expect.any(Buffer), + expect.objectContaining({ + format: "jpeg", + quality: 61, + resize: { width: 1456, enlarge: false }, + }), + ); + }); + + it.each(["png", "jpeg"] as const)("encodes %s snapshots", async (format) => { + const { driver } = createDriver(); + const { processor, encode } = createProcessor(); + const { snapshot } = commandSet(driver, processor); + + const payload = JSON.parse( + await snapshot.handle(JSON.stringify({ maxWidth: 2000, format })), + ) as { format: string; base64: string }; + + expect(payload.format).toBe(format); + expect(Buffer.from(payload.base64, "base64").toString()).toBe(`${format}-encoded`); + expect(encode).toHaveBeenCalledWith(expect.any(Buffer), expect.objectContaining({ format })); + }); + + it("returns the native PNG without re-encoding when no resize is needed", async () => { + const { driver } = createDriver(); + const { processor, encode } = createProcessor(); + const { snapshot } = commandSet(driver, processor); + + const payload = JSON.parse( + await snapshot.handle(JSON.stringify({ maxWidth: 4000, format: "png" })), + ) as { base64: string; width: number; height: number }; + + expect(Buffer.from(payload.base64, "base64").toString()).toBe("native-png"); + expect(payload).toMatchObject({ width: 3840, height: 2160 }); + expect(encode).not.toHaveBeenCalled(); + }); + + it("rejects non-primary screen indexes", async () => { + const { driver } = createDriver(); + const { snapshot } = commandSet(driver); + await expect(snapshot.handle('{"screenIndex":1}')).rejects.toThrow( + "COMPUTER_UNSUPPORTED_DISPLAY", + ); + }); + + it("refuses capture when screen and screenshot geometry differ", async () => { + // Guards the scalePoint invariant: input is native-pixel, so a capture whose + // screen geometry differs from its screenshot pixels would mis-target. + const { driver } = createDriver({ + desktop: () => desktopResult({ screen_width: 2560, screen_height: 1440 }), + }); + const { snapshot } = commandSet(driver); + await expect(snapshot.handle("{}")).rejects.toThrow( + "COMPUTER_UNSUPPORTED_DISPLAY: cua-driver reported capture and screen geometry", + ); + }); + + it("rotates the frame token when captured display geometry changes", async () => { + let width = 3840; + const { driver } = createDriver({ + desktop: () => + desktopResult({ + screenshot_width: width, + screen_width: width, + }), + }); + const { snapshot } = commandSet(driver); + const first = JSON.parse(await snapshot.handle('{"format":"png","maxWidth":4000}')) as { + displayFrameId: string; + }; + width = 2560; + const second = JSON.parse(await snapshot.handle('{"format":"png","maxWidth":4000}')) as { + displayFrameId: string; + }; + expect(second.displayFrameId).not.toBe(first.displayFrameId); + }); +}); + +describe("cua-computer computer.act", () => { + it("maps all supported pointer actions to desktop-scope driver calls", async () => { + const { driver, calls } = createDriver(); + const { act, frameId, refWidth } = await issueFrameFor(driver); + calls.length = 0; + const base = { displayFrameId: frameId, refWidth, x: 960, y: 540 }; + const cases = [ + ["left_click", "click", { x: 1920, y: 1080, button: "left", count: 1 }], + ["right_click", "click", { x: 1920, y: 1080, button: "right", count: 1 }], + ["middle_click", "click", { x: 1920, y: 1080, button: "middle", count: 1 }], + ["double_click", "click", { x: 1920, y: 1080, button: "left", count: 2 }], + ["triple_click", "click", { x: 1920, y: 1080, button: "left", count: 3 }], + ["mouse_move", "move_cursor", { x: 1920, y: 1080 }], + ["left_click_drag", "drag", { from_x: 200, from_y: 400, to_x: 1920, to_y: 1080 }], + ["scroll", "scroll", { x: 1920, y: 1080, direction: "down", amount: 50, by: "line" }], + ] as const; + + for (const [action, tool, expected] of cases) { + await act.handle( + JSON.stringify({ + action, + ...base, + ...(action === "left_click_drag" ? { fromX: 100, fromY: 200 } : {}), + ...(action === "scroll" ? { scrollDirection: "down", scrollAmount: 99 } : {}), + }), + ); + const call = calls.at(-1); + expect(call?.name).toBe(tool); + expect(call?.args).toMatchObject({ ...expected, scope: "desktop" }); + } + }); + + it("normalizes click modifiers and keyboard chords", async () => { + const { driver, calls } = createDriver(); + const { act, frameId, refWidth } = await issueFrameFor(driver, "win32"); + calls.length = 0; + + await act.handle( + JSON.stringify({ + action: "left_click", + displayFrameId: frameId, + refWidth, + x: 10, + y: 20, + modifiers: "cmd+Control", + }), + ); + await act.handle(JSON.stringify({ action: "key", keys: "super+shift+Return" })); + + expect(calls.find((call) => call.name === "click")?.args).toMatchObject({ + modifier: ["meta", "ctrl"], + }); + expect(calls.find((call) => call.name === "press_key")?.args).toEqual({ + key: "enter", + modifiers: ["meta", "shift"], + scope: "desktop", + }); + }); + + it("maps type without geometry calls and rejects the wire-less wait action", async () => { + const { driver, calls } = createDriver(); + const { act } = commandSet(driver); + + await act.handle(JSON.stringify({ action: "type", text: "hello" })); + // Core sleeps locally for wait and never sends it over the wire; the + // fulfiller must reject it so the computer.act contract stays uniform. + await expect(act.handle(JSON.stringify({ action: "wait", durationMs: 25 }))).rejects.toThrow( + /COMPUTER_INVALID_REQUEST/, + ); + + expect(calls).toEqual([{ name: "type_text", args: { text: "hello", scope: "desktop" } }]); + }); + + it("scrolls at frame-authorized coordinates", async () => { + const { driver, calls } = createDriver(); + const { act, frameId, refWidth } = await issueFrameFor(driver); + + await act.handle( + JSON.stringify({ + action: "scroll", + scrollDirection: "up", + scrollAmount: 2, + displayFrameId: frameId, + refWidth, + x: 960, + y: 540, + }), + ); + + expect(calls.at(-1)).toEqual({ + name: "scroll", + args: { direction: "up", amount: 2, by: "line", x: 1920, y: 1080, scope: "desktop" }, + }); + }); + + it("rejects coordinate-less scroll instead of guessing the cursor point", async () => { + const { driver, calls } = createDriver(); + const { act } = commandSet(driver); + + await expect( + act.handle(JSON.stringify({ action: "scroll", scrollDirection: "up", scrollAmount: 2 })), + ).rejects.toThrow("COMPUTER_STALE_FRAME"); + expect(calls.some((call) => call.name === "get_cursor_position")).toBe(false); + }); + + it.each([0, -3])("rejects non-positive scroll amount %s instead of scrolling", async (amount) => { + const { driver, calls } = createDriver(); + const { act, frameId, refWidth } = await issueFrameFor(driver); + await expect( + act.handle( + JSON.stringify({ + action: "scroll", + scrollDirection: "up", + scrollAmount: amount, + displayFrameId: frameId, + refWidth, + x: 10, + y: 10, + }), + ), + ).rejects.toThrow("COMPUTER_INVALID_REQUEST"); + expect(calls.some((call) => call.name === "scroll")).toBe(false); + }); + + it("performs an unmodified drag with scaled coordinates", async () => { + const { driver, calls } = createDriver(); + const { act, frameId, refWidth } = await issueFrameFor(driver); + + await act.handle( + JSON.stringify({ + action: "left_click_drag", + displayFrameId: frameId, + refWidth, + fromX: 0, + fromY: 0, + x: 960, + y: 540, + }), + ); + + expect(calls.at(-1)).toEqual({ + name: "drag", + args: { from_x: 0, from_y: 0, to_x: 1920, to_y: 1080, scope: "desktop" }, + }); + }); + + it("clamps drag duration to the driver's supported maximum", async () => { + const { driver, calls } = createDriver(); + const { act, frameId, refWidth } = await issueFrameFor(driver); + + await act.handle( + JSON.stringify({ + action: "left_click_drag", + displayFrameId: frameId, + refWidth, + fromX: 0, + fromY: 0, + x: 960, + y: 540, + durationMs: 15_000, + }), + ); + + expect(calls.at(-1)).toEqual({ + name: "drag", + args: { from_x: 0, from_y: 0, to_x: 1920, to_y: 1080, scope: "desktop", duration_ms: 10_000 }, + }); + }); + + it("rejects modifier-held drags that cua-driver silently drops", async () => { + const { driver } = createDriver(); + const { act, frameId, refWidth } = await issueFrameFor(driver); + + await expect( + act.handle( + JSON.stringify({ + action: "left_click_drag", + displayFrameId: frameId, + refWidth, + fromX: 0, + fromY: 0, + x: 960, + y: 540, + modifiers: "shift", + }), + ), + ).rejects.toThrow("modifier-held drag is unsupported"); + }); + + it("rejects modifier actions that cua-driver cannot preserve", async () => { + const { driver } = createDriver(); + const { act, frameId, refWidth } = await issueFrameFor(driver); + await expect( + act.handle( + JSON.stringify({ + action: "left_click", + displayFrameId: frameId, + refWidth, + x: 1, + y: 1, + modifiers: "shift", + }), + ), + ).rejects.toThrow("modifier-held clicks are unsupported"); + await expect( + act.handle(JSON.stringify({ action: "scroll", scrollDirection: "down", modifiers: "ctrl" })), + ).rejects.toThrow("modifier-held scroll is unsupported"); + }); + + it.each(["hold_key", "left_mouse_down", "left_mouse_up"])( + "rejects unsupported %s", + async (action) => { + const { driver } = createDriver(); + const { act } = commandSet(driver); + await expect(act.handle(JSON.stringify({ action }))).rejects.toThrow( + `COMPUTER_UNSUPPORTED_ACTION: ${action}`, + ); + }, + ); + + it("rejects wrong ids, geometry drift, and reference-width drift", async () => { + let currentSize = screenSizeResult(); + const { driver } = createDriver({ + callTool: async (name) => { + if (name === "get_desktop_state") { + return desktopResult(); + } + if (name === "get_screen_size") { + return currentSize; + } + return { content: [] }; + }, + }); + + const wrong = await issueFrameFor(driver); + await expect( + wrong.act.handle( + JSON.stringify({ + action: "left_click", + displayFrameId: "cua:v1:wrong", + refWidth: wrong.refWidth, + x: 1, + y: 1, + }), + ), + ).rejects.toThrow("COMPUTER_STALE_FRAME"); + + const drift = await issueFrameFor(driver); + currentSize = screenSizeResult({ width: 2560 }); + await expect( + drift.act.handle( + JSON.stringify({ + action: "mouse_move", + displayFrameId: drift.frameId, + refWidth: drift.refWidth, + x: 1, + y: 1, + }), + ), + ).rejects.toThrow("COMPUTER_STALE_FRAME"); + + currentSize = screenSizeResult(); + const width = await issueFrameFor(driver); + await expect( + width.act.handle( + JSON.stringify({ + action: "mouse_move", + displayFrameId: width.frameId, + refWidth: width.refWidth + 1, + x: 1, + y: 1, + }), + ), + ).rejects.toThrow("COMPUTER_STALE_FRAME"); + + const missing = await issueFrameFor(driver); + await expect( + missing.act.handle( + JSON.stringify({ + action: "mouse_move", + displayFrameId: missing.frameId, + x: 1, + y: 1, + }), + ), + ).rejects.toThrow("COMPUTER_STALE_FRAME"); + }); + + it("rejects frames across driver reconnects", async () => { + let generation = 1; + const { driver } = createDriver({ + generation: () => generation, + callTool: async (name) => { + if (name === "get_desktop_state") { + return desktopResult(); + } + if (name === "get_screen_size") { + generation = 2; + return screenSizeResult(); + } + return { content: [] }; + }, + }); + const { act, frameId, refWidth } = await issueFrameFor(driver); + await expect( + act.handle( + JSON.stringify({ action: "mouse_move", displayFrameId: frameId, refWidth, x: 1, y: 1 }), + ), + ).rejects.toThrow("the computer driver reconnected"); + }); + + it("rejects coordinates outside the delivered primary-display frame", async () => { + const { driver } = createDriver(); + const { act, frameId, refWidth } = await issueFrameFor(driver); + await expect( + act.handle( + JSON.stringify({ + action: "left_click", + displayFrameId: frameId, + refWidth, + x: refWidth, + y: 0, + }), + ), + ).rejects.toThrow("outside the captured primary-display frame"); + }); + + it("preserves structured driver refusal errors", async () => { + const { driver } = createDriver({ + callTool: async (name) => { + if (name === "get_desktop_state") { + return desktopResult(); + } + if (name === "get_screen_size") { + return screenSizeResult(); + } + throw new Error("COMPUTER_REFUSED_background_unavailable: desktop unavailable"); + }, + }); + const { act, frameId, refWidth } = await issueFrameFor(driver); + await expect( + act.handle( + JSON.stringify({ action: "left_click", displayFrameId: frameId, refWidth, x: 1, y: 1 }), + ), + ).rejects.toThrow("COMPUTER_REFUSED_background_unavailable"); + }); + + it("serializes interleaved action calls", async () => { + let releaseFirst = () => {}; + const firstPending = new Promise((resolve) => { + releaseFirst = resolve; + }); + const started: string[] = []; + const { driver } = createDriver({ + callTool: async (name, args) => { + if (name === "type_text") { + started.push(String(args.text)); + if (args.text === "first") { + await firstPending; + } + } + return { content: [] }; + }, + }); + const { act } = commandSet(driver); + + const first = act.handle(JSON.stringify({ action: "type", text: "first" })); + const second = act.handle(JSON.stringify({ action: "type", text: "second" })); + await vi.waitFor(() => expect(started).toEqual(["first"])); + releaseFirst(); + await Promise.all([first, second]); + expect(started).toEqual(["first", "second"]); + }); + + it("rejects unknown key and modifier names", async () => { + const { driver } = createDriver(); + const { act, frameId, refWidth } = await issueFrameFor(driver); + await expect(act.handle(JSON.stringify({ action: "key", keys: "hyper+x" }))).rejects.toThrow( + "COMPUTER_UNSUPPORTED_KEY", + ); + await expect( + act.handle( + JSON.stringify({ + action: "left_click", + displayFrameId: frameId, + refWidth, + x: 1, + y: 1, + modifiers: "hyper", + }), + ), + ).rejects.toThrow("COMPUTER_UNSUPPORTED_KEY"); + }); +}); + +describe("cua-computer availability", () => { + it.each([ + ["darwin", true, false], + ["linux", true, true], + ["linux", false, false], + ] as const)("returns %s availability with binary=%s", (platform, binary, expected) => { + const { driver } = createDriver({ available: binary }); + const command = createCuaComputerCommands({ + platform, + driver, + imageProcessor: createProcessor().processor, + })[0]; + expect(command?.isAvailable?.({ config: {}, env: {} })).toBe(expected); + }); +}); diff --git a/extensions/cua-computer/src/commands.ts b/extensions/cua-computer/src/commands.ts new file mode 100644 index 000000000000..7814ef8a636a --- /dev/null +++ b/extensions/cua-computer/src/commands.ts @@ -0,0 +1,462 @@ +import fs from "node:fs"; +import path from "node:path"; +import type { OpenClawPluginNodeHostCommand } from "openclaw/plugin-sdk/plugin-entry"; +import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path"; +import { createRastermill } from "rastermill"; +import { z } from "zod"; +import { + ComputerActParamsSchema, + normalizeModifiers, + parseKeyChord, + scalePoint, + type ComputerActParams, +} from "./actions.js"; +import { CuaDriverClient, type CuaDriver, type CuaToolResult } from "./driver-client.js"; +import { + issueFrame, + verifyFrame, + verifyReferenceWidth, + type CuaDesktopGeometry, + type CuaFrameState, + type CuaLastFrame, + type CuaScreenSize, +} from "./frame.js"; + +const AVAILABILITY_POLL_MS = 5_000; +// Rastermill enforces inputPixels before resizing, so this must clear the native +// capture, not the delivered frame. 8K (7680x4320 = ~33.2M) is a valid primary +// display; budget above it so full-resolution snapshots reach the downscaler. +const MAX_IMAGE_PIXELS = 40_000_000; + +const SnapshotParamsSchema = z.strictObject({ + screenIndex: z.number().int().nonnegative().optional(), + maxWidth: z.number().int().positive().optional(), + quality: z.number().finite().optional(), + format: z.enum(["jpeg", "png"]).optional(), +}); + +const DesktopStateSchema = z.object({ + platform: z.string().min(1), + display: z.string().min(1), + screenshot_width: z.number().int().positive(), + screenshot_height: z.number().int().positive(), + screen_width: z.number().int().positive(), + screen_height: z.number().int().positive(), + scale_factor: z.number().positive(), +}); + +const ScreenSizeSchema = z.object({ + width: z.number().int().positive(), + height: z.number().int().positive(), + scale_factor: z.number().positive(), +}); + +type ImageProcessor = { + encode( + input: Buffer, + options: { + format: "jpeg" | "png"; + quality?: number; + resize?: { width: number; enlarge: false }; + }, + ): Promise<{ data: Buffer; width: number; height: number }>; +}; + +type CuaComputerCommandsOptions = { + driverPath?: string; + platform?: NodeJS.Platform; + env?: NodeJS.ProcessEnv; + driver?: CuaDriver; + imageProcessor?: ImageProcessor; + setInterval?: typeof setInterval; + clearInterval?: typeof clearInterval; +}; + +class PromiseQueue { + private tail: Promise = Promise.resolve(); + + async run(operation: () => Promise): Promise { + const previous = this.tail; + let release = () => {}; + this.tail = new Promise((resolve) => { + release = resolve; + }); + await previous; + try { + return await operation(); + } finally { + release(); + } + } +} + +function parseParams(schema: z.ZodType, paramsJSON: string | null | undefined): T { + let value: unknown; + try { + value = JSON.parse(paramsJSON ?? "{}"); + } catch { + throw new Error("COMPUTER_INVALID_REQUEST: params must be valid JSON"); + } + const parsed = schema.safeParse(value); + if (!parsed.success) { + throw new Error( + `COMPUTER_INVALID_REQUEST: ${parsed.error.issues[0]?.message ?? "invalid params"}`, + ); + } + return parsed.data; +} + +function assertPrimaryDisplay(screenIndex: number | undefined): void { + if (screenIndex !== undefined && screenIndex !== 0) { + throw new Error( + "COMPUTER_UNSUPPORTED_DISPLAY: cua-driver controls only the primary display (screenIndex 0)", + ); + } +} + +function structuredContent(result: CuaToolResult, tool: string): Record { + if (!result.structuredContent) { + throw new Error(`COMPUTER_DRIVER_ERROR: ${tool} returned no structuredContent`); + } + return result.structuredContent; +} + +function desktopGeometry(result: CuaToolResult): CuaDesktopGeometry { + const parsed = DesktopStateSchema.safeParse(structuredContent(result, "get_desktop_state")); + if (!parsed.success) { + throw new Error("COMPUTER_DRIVER_ERROR: invalid get_desktop_state geometry"); + } + return { + platform: parsed.data.platform, + display: parsed.data.display, + screenWidth: parsed.data.screen_width, + screenHeight: parsed.data.screen_height, + scaleFactor: parsed.data.scale_factor, + screenshotWidth: parsed.data.screenshot_width, + screenshotHeight: parsed.data.screenshot_height, + }; +} + +function desktopPng(result: CuaToolResult): Buffer { + const image = result.content.find( + (entry): entry is { type: "image"; data: string; mimeType: string } => + entry.type === "image" && typeof entry.data === "string" && entry.mimeType === "image/png", + ); + if (!image) { + throw new Error("COMPUTER_DRIVER_ERROR: get_desktop_state returned no PNG image"); + } + return Buffer.from(image.data, "base64"); +} + +function screenSize(result: CuaToolResult): CuaScreenSize { + const parsed = ScreenSizeSchema.safeParse(structuredContent(result, "get_screen_size")); + if (!parsed.success) { + throw new Error("COMPUTER_DRIVER_ERROR: invalid get_screen_size geometry"); + } + return { + width: parsed.data.width, + height: parsed.data.height, + scaleFactor: parsed.data.scale_factor, + }; +} + +function resolveImageCommand(command: string, env: NodeJS.ProcessEnv): string | null { + const names = + process.platform === "win32" && !path.extname(command) + ? [command, `${command}.exe`, `${command}.cmd`] + : [command]; + for (const entry of (env.PATH ?? "").split(path.delimiter).filter(Boolean)) { + for (const name of names) { + const candidate = path.resolve(entry, name); + try { + fs.accessSync(candidate, fs.constants.X_OK); + return candidate; + } catch { + // Continue through PATH. + } + } + } + return null; +} + +function createImageProcessor(env: NodeJS.ProcessEnv): ImageProcessor { + return createRastermill({ + execution: "auto", + limits: { inputPixels: MAX_IMAGE_PIXELS, outputPixels: MAX_IMAGE_PIXELS }, + temp: { rootDir: resolvePreferredOpenClawTmpDir(), prefix: "openclaw-cua-computer-" }, + commandResolver: (command) => resolveImageCommand(command, env), + }); +} + +function clickArgs( + platform: NodeJS.Platform, + frame: CuaLastFrame, + params: ComputerActParams, + button: "left" | "right" | "middle", + count: 1 | 2 | 3, +): Record { + const point = scalePoint(frame, params.x, params.y, params.action); + const modifiers = normalizeModifiers(params.modifiers); + if (platform === "linux" && modifiers.length > 0) { + throw new Error( + "COMPUTER_UNSUPPORTED_ACTION: modifier-held clicks are unsupported by cua-driver on Linux", + ); + } + return { + ...point, + scope: "desktop", + button, + count, + ...(modifiers.length > 0 ? { modifier: modifiers } : {}), + }; +} + +async function currentFrame( + driver: CuaDriver, + frameState: CuaFrameState, + params: ComputerActParams, +): Promise { + const current = screenSize(await driver.callTool("get_screen_size", {})); + if (driver.generation !== frameState.generation) { + frameState.lastFrame = undefined; + throw new Error("COMPUTER_STALE_FRAME: the computer driver reconnected; take a new screenshot"); + } + const frame = verifyFrame(frameState, params.displayFrameId, current); + verifyReferenceWidth(frameState, frame, params.refWidth); + return frame; +} + +async function handleAct( + driver: CuaDriver, + frameState: CuaFrameState, + params: ComputerActParams, + platform: NodeJS.Platform, +): Promise { + assertPrimaryDisplay(params.screenIndex); + // `wait` never reaches the wire: core sleeps locally and the Swift wire enum + // has no wait case, so accepting it here would fork the computer.act contract. + if ( + params.action === "hold_key" || + params.action === "left_mouse_down" || + params.action === "left_mouse_up" + ) { + // Upstream has no desktop keyboard-down API, and its Linux mouse hold tools + // are window-only, so these actions cannot preserve desktop-scope semantics. + throw new Error(`COMPUTER_UNSUPPORTED_ACTION: ${params.action}`); + } + + // Every action uses scope:"desktop", a global SendInput/XTest/wayland_desktop + // injection that is inherently foreground and ignores delivery_mode (that + // background-vs-foreground contract is window-targeted only). We deliberately + // never send delivery_mode. + switch (params.action) { + case "type": { + if (!params.text) { + throw new Error("COMPUTER_INVALID_REQUEST: text is required for type"); + } + await driver.callTool("type_text", { text: params.text, scope: "desktop" }); + break; + } + case "key": { + // press_key applies the modifier array on every backend: X11 via XTest, + // and native Wayland by internally promoting a modifier chord to + // hotkey_focused. No separate hotkey call is needed for chords. + const chord = parseKeyChord(params.keys); + await driver.callTool("press_key", { + key: chord.key, + modifiers: chord.modifiers, + scope: "desktop", + }); + break; + } + case "scroll": { + if (!params.scrollDirection) { + throw new Error("COMPUTER_INVALID_REQUEST: scrollDirection is required for scroll"); + } + if (normalizeModifiers(params.modifiers).length > 0) { + throw new Error( + "COMPUTER_UNSUPPORTED_ACTION: modifier-held scroll is unsupported by cua-driver 0.10.x", + ); + } + // Desktop-scope scroll requires explicit coordinates, and they must be + // frame-authorized like clicks. We deliberately do not synthesize a point + // from get_cursor_position: that mixes cursor and capture coordinate + // spaces across X11/Wayland/Windows and would scroll an unverified target. + const frame = await currentFrame(driver, frameState, params); + const point = scalePoint(frame, params.x, params.y, params.action); + await driver.callTool("scroll", { + direction: params.scrollDirection, + // Schema guarantees a positive amount; cap at the driver's max of 50. + amount: Math.min(50, params.scrollAmount ?? 3), + by: "line", + ...point, + scope: "desktop", + }); + break; + } + default: { + const frame = await currentFrame(driver, frameState, params); + switch (params.action) { + case "left_click": + await driver.callTool("click", clickArgs(platform, frame, params, "left", 1)); + break; + case "right_click": + await driver.callTool("click", clickArgs(platform, frame, params, "right", 1)); + break; + case "middle_click": + await driver.callTool("click", clickArgs(platform, frame, params, "middle", 1)); + break; + case "double_click": + await driver.callTool("click", clickArgs(platform, frame, params, "left", 2)); + break; + case "triple_click": + await driver.callTool("click", clickArgs(platform, frame, params, "left", 3)); + break; + case "mouse_move": { + const point = scalePoint(frame, params.x, params.y, params.action); + await driver.callTool("move_cursor", { ...point, scope: "desktop" }); + break; + } + case "left_click_drag": { + const from = scalePoint(frame, params.fromX, params.fromY, "drag start"); + const to = scalePoint(frame, params.x, params.y, "drag end"); + // cua-driver 0.10 accepts `modifier` in the drag schema but its + // desktop-scope branch never reads it (Windows impl_.rs drag desktop + // path uses only coords/duration/steps/button), so a Shift/Ctrl-drag + // would silently become a plain drag. Refuse instead of misfiring. + if (normalizeModifiers(params.modifiers).length > 0) { + throw new Error( + "COMPUTER_UNSUPPORTED_ACTION: modifier-held drag is unsupported by cua-driver 0.10.x", + ); + } + await driver.callTool("drag", { + from_x: from.x, + from_y: from.y, + to_x: to.x, + to_y: to.y, + scope: "desktop", + // cua-driver caps drag duration_ms at 10_000; clamp so a longer + // request runs at the max instead of being rejected at the MCP edge. + ...(params.durationMs === undefined + ? {} + : { duration_ms: Math.min(10_000, params.durationMs) }), + }); + break; + } + default: + throw new Error("COMPUTER_UNSUPPORTED_ACTION: unknown action"); + } + } + } + return JSON.stringify({ ok: true }); +} + +export function createCuaComputerCommands( + options: CuaComputerCommandsOptions = {}, +): OpenClawPluginNodeHostCommand[] { + const platform = options.platform ?? process.platform; + const env = options.env ?? process.env; + const driver = + options.driver ?? new CuaDriverClient({ driverPath: options.driverPath, platform, env }); + const imageProcessor = options.imageProcessor ?? createImageProcessor(env); + const queue = new PromiseQueue(); + const frameState: CuaFrameState = { generation: driver.generation }; + const interval = options.setInterval ?? setInterval; + const clear = options.clearInterval ?? clearInterval; + const isSupportedPlatform = platform === "linux" || platform === "win32"; + const isAvailable = () => isSupportedPlatform && driver.isAvailable(); + + const snapshot: OpenClawPluginNodeHostCommand = { + command: "screen.snapshot", + cap: "screen", + dangerous: false, + isAvailable, + watchAvailability: (_context, onChange) => { + let knownAvailable = isAvailable(); + const timer = interval(() => { + driver.resetAvailabilityCache(); + const available = isAvailable(); + if (available !== knownAvailable) { + knownAvailable = available; + onChange(); + } + }, AVAILABILITY_POLL_MS); + timer.unref?.(); + return () => { + clear(timer); + void driver.dispose(); + }; + }, + handle: async (paramsJSON) => + await queue.run(async () => { + if (!isSupportedPlatform) { + throw new Error("COMPUTER_DRIVER_UNAVAILABLE: cua-computer supports Windows and Linux"); + } + const params = parseParams(SnapshotParamsSchema, paramsJSON); + assertPrimaryDisplay(params.screenIndex); + const format = params.format ?? "jpeg"; + const maxWidth = params.maxWidth ?? (format === "png" ? 900 : 1_600); + const quality = Math.min(1, Math.max(0.05, params.quality ?? 0.72)); + const desktop = await driver.callTool("get_desktop_state", {}); + const geometry = desktopGeometry(desktop); + // cua-driver desktop input consumes native get_desktop_state PNG pixels, + // and on every supported backend the driver reports screen geometry in + // that same physical-pixel space (Windows PMv2, Linux X11/Wayland). If a + // capture ever diverges from screen geometry, our screenshot->native + // scaling would mis-target input, so refuse rather than click blind. + if ( + geometry.screenWidth !== geometry.screenshotWidth || + geometry.screenHeight !== geometry.screenshotHeight + ) { + throw new Error( + "COMPUTER_UNSUPPORTED_DISPLAY: cua-driver reported capture and screen geometry in different pixel spaces", + ); + } + const nativePng = desktopPng(desktop); + let encoded = nativePng; + let width = geometry.screenshotWidth; + let height = geometry.screenshotHeight; + if (format === "jpeg" || width > maxWidth) { + const result = await imageProcessor.encode(nativePng, { + format, + ...(format === "jpeg" ? { quality: Math.round(quality * 100) } : {}), + ...(width > maxWidth ? { resize: { width: maxWidth, enlarge: false } } : {}), + }); + encoded = result.data; + width = result.width; + height = result.height; + } + frameState.generation = driver.generation; + const displayFrameId = issueFrame(frameState, geometry, { width, height }); + return JSON.stringify({ + format, + base64: encoded.toString("base64"), + displayFrameId, + screenIndex: 0, + width, + height, + }); + }), + }; + + const act: OpenClawPluginNodeHostCommand = { + command: "computer.act", + cap: "computer", + dangerous: true, + isAvailable, + handle: async (paramsJSON) => + await queue.run(async () => { + if (!isSupportedPlatform) { + throw new Error("COMPUTER_DRIVER_UNAVAILABLE: cua-computer supports Windows and Linux"); + } + return await handleAct( + driver, + frameState, + parseParams(ComputerActParamsSchema, paramsJSON), + platform, + ); + }), + }; + + return [snapshot, act]; +} diff --git a/extensions/cua-computer/src/driver-client.test.ts b/extensions/cua-computer/src/driver-client.test.ts new file mode 100644 index 000000000000..250eea200b23 --- /dev/null +++ b/extensions/cua-computer/src/driver-client.test.ts @@ -0,0 +1,480 @@ +import type { ChildProcess } from "node:child_process"; +import type { Transport } from "@modelcontextprotocol/sdk/shared/transport.js"; +import { describe, expect, it, vi } from "vitest"; +import { CuaDriverClient } from "./driver-client.js"; + +function fakeTransport(): Transport { + return { + start: vi.fn(async () => {}), + send: vi.fn(async () => {}), + close: vi.fn(async () => {}), + }; +} + +function fakeClient(options: { + name?: string; + version?: string; + capabilityVersion?: string; + schemaVersion?: string; + connect?: () => Promise; + result?: unknown; + callToolThrows?: boolean; +}) { + return { + connect: vi.fn(options.connect ?? (async () => {})), + getServerVersion: () => ({ + name: options.name ?? "cua-driver", + version: options.version ?? "0.10.4", + }), + listTools: vi.fn(async () => ({ + tools: [], + capability_version: options.capabilityVersion ?? "1", + schema_version: options.schemaVersion ?? "1", + })), + callTool: vi.fn(async () => { + if (options.callToolThrows) { + throw new Error("transport closed"); + } + return options.result ?? { content: [{ type: "text", text: "ok" }] }; + }), + close: vi.fn(async () => {}), + }; +} + +function createClient(client: ReturnType) { + return new CuaDriverClient({ + driverPath: "/opt/bin/cua-driver", + access: () => {}, + clientFactory: () => client, + transportFactory: fakeTransport, + }); +} + +describe("CuaDriverClient version gate", () => { + it("accepts cua-driver 0.10.x with capability and schema version 1", async () => { + const driver = createClient(fakeClient({ version: "0.10.4" })); + await expect(driver.callTool("get_screen_size", {})).resolves.toMatchObject({ + content: [{ type: "text", text: "ok" }], + }); + expect(driver.generation).toBe(1); + }); + + it.each([ + [{ version: "0.11.0" }, "cua-driver@0.11.0"], + [{ name: "other-driver" }, "other-driver@0.10.4"], + [{ capabilityVersion: "2" }, "capability_version=2"], + [{ schemaVersion: "2" }, "schema_version=2"], + ])("rejects unsupported initialize/list result %#", async (options, found) => { + const driver = createClient(fakeClient(options)); + await expect(driver.callTool("get_screen_size", {})).rejects.toThrow(found); + expect(driver.isAvailable()).toBe(false); + }); + + it("re-probes and recovers after a corrected driver replaces an unsupported one", async () => { + const clients = [fakeClient({ version: "0.11.0" }), fakeClient({ version: "0.10.4" })]; + let clock = 1_000; + const driver = new CuaDriverClient({ + driverPath: "/opt/bin/cua-driver", + access: () => {}, + clientFactory: () => clients.shift() ?? fakeClient({ version: "0.10.4" }), + transportFactory: fakeTransport, + now: () => clock, + }); + + await expect(driver.callTool("get_screen_size", {})).rejects.toThrow("cua-driver@0.11.0"); + // Within the re-probe window the verdict is cached: unavailable, no reconnect. + clock += 10_000; + expect(driver.isAvailable()).toBe(false); + // After the window, the corrected driver is re-probed and accepted. + clock += 25_000; + expect(driver.isAvailable()).toBe(true); + await expect(driver.callTool("get_screen_size", {})).resolves.toMatchObject({ + content: [{ type: "text", text: "ok" }], + }); + }); +}); + +describe("CuaDriverClient process contract", () => { + it("forwards only allowlisted env, opt-outs, and the Wayland setting, dropping secrets", async () => { + let transportParams: { env?: Record; stderr?: unknown } | undefined; + const driver = new CuaDriverClient({ + driverPath: "/opt/bin/cua-driver", + env: { + PATH: "/opt/bin", + DISPLAY: ":0", + XDG_RUNTIME_DIR: "/run/user/1000", + LC_ALL: "en_US.UTF-8", + CUA_DRIVER_RS_ENABLE_WAYLAND: "1", + OPENAI_API_KEY: "sk-should-not-leak", + ANTHROPIC_API_KEY: "should-not-leak", + SLACK_BOT_TOKEN: "xoxb-should-not-leak", + CUA_API_KEY: "cua-cloud-should-not-leak", + }, + access: () => {}, + clientFactory: () => fakeClient({}), + transportFactory: (params) => { + transportParams = params; + return fakeTransport(); + }, + }); + + await driver.callTool("get_screen_size", {}); + + expect(transportParams?.env).toMatchObject({ + PATH: "/opt/bin", + DISPLAY: ":0", + XDG_RUNTIME_DIR: "/run/user/1000", + LC_ALL: "en_US.UTF-8", + CUA_DRIVER_RS_TELEMETRY_ENABLED: "false", + CUA_DRIVER_RS_UPDATE_CHECK: "false", + CUA_DRIVER_RS_ENABLE_WAYLAND: "1", + }); + expect(transportParams?.env).not.toHaveProperty("OPENAI_API_KEY"); + expect(transportParams?.env).not.toHaveProperty("ANTHROPIC_API_KEY"); + expect(transportParams?.env).not.toHaveProperty("SLACK_BOT_TOKEN"); + // The CUA_ namespace holds cloud credentials, so it is not prefix-allowed. + expect(transportParams?.env).not.toHaveProperty("CUA_API_KEY"); + expect(transportParams?.stderr).toBe("ignore"); + }); + + it("starts serve and connects on the first readiness poll", async () => { + const first = fakeClient({ + connect: async () => { + throw new Error("daemon absent"); + }, + }); + const second = fakeClient({}); + const clients = [first, second]; + const kill = vi.fn(); + const child = { + exitCode: null, + once: vi.fn(), + unref: vi.fn(), + kill, + } as unknown as ChildProcess; + const spawnProcess = vi.fn(() => child); + const driver = new CuaDriverClient({ + driverPath: "/opt/bin/cua-driver", + env: { PATH: "/opt/bin" }, + access: () => {}, + clientFactory: () => clients.shift() ?? second, + transportFactory: fakeTransport, + spawn: spawnProcess as unknown as typeof import("node:child_process").spawn, + sleep: async () => {}, + }); + + await driver.callTool("get_screen_size", {}); + expect(spawnProcess).toHaveBeenCalledWith( + "/opt/bin/cua-driver", + ["serve"], + expect.objectContaining({ + detached: true, + env: expect.objectContaining({ + CUA_DRIVER_RS_TELEMETRY_ENABLED: "false", + CUA_DRIVER_RS_UPDATE_CHECK: "false", + }), + }), + ); + await driver.dispose(); + // The shared machine daemon must outlive our client; dispose closes the mcp + // session but never kills serve, so other cua-driver clients stay connected. + expect(kill).not.toHaveBeenCalled(); + }); + + it("keeps polling through a slow daemon start and records the backoff delays", async () => { + const failing = () => + fakeClient({ + connect: async () => { + throw new Error("daemon still starting"); + }, + }); + const clients = [failing(), failing(), failing(), fakeClient({})]; + const delays: number[] = []; + const child = { + exitCode: null, + once: vi.fn(), + unref: vi.fn(), + kill: vi.fn(), + } as unknown as ChildProcess; + const driver = new CuaDriverClient({ + driverPath: "/opt/bin/cua-driver", + env: { PATH: "/opt/bin" }, + access: () => {}, + clientFactory: () => clients.shift() ?? fakeClient({}), + transportFactory: fakeTransport, + spawn: vi.fn(() => child) as unknown as typeof import("node:child_process").spawn, + sleep: async (durationMs) => { + delays.push(durationMs); + }, + }); + + await driver.callTool("get_screen_size", {}); + expect(delays).toEqual([250, 500, 1_000]); + }); + + it("keeps polling the full budget even after the spawned child exits", async () => { + const delays: number[] = []; + const child = { + exitCode: null, + once: vi.fn(), + unref: vi.fn(), + kill: vi.fn(), + } as unknown as ChildProcess; + const driver = new CuaDriverClient({ + driverPath: "/opt/bin/cua-driver", + env: { PATH: "/opt/bin" }, + access: () => {}, + clientFactory: () => + fakeClient({ + connect: async () => { + throw new Error("daemon absent"); + }, + }), + transportFactory: fakeTransport, + spawn: vi.fn(() => child) as unknown as typeof import("node:child_process").spawn, + sleep: async (durationMs) => { + delays.push(durationMs); + (child as { exitCode: number | null }).exitCode = 3; + }, + }); + + await expect(driver.callTool("get_screen_size", {})).rejects.toThrow( + /COMPUTER_DRIVER_UNAVAILABLE: cua-driver daemon did not become ready in time/, + ); + // Child exit must not short-circuit the schedule. + expect(delays).toEqual([250, 500, 1_000, 2_000, 3_000, 3_000]); + }); + + it("respawns the daemon when the remembered child was signal-terminated", async () => { + // Two connect cycles: cycle 1 spawns child0 and caches a session whose + // callTool then breaks; cycle 2 must spawn again because child0 was killed + // by signal (exitCode null, signalCode set) rather than a clean exit. + const clients = [ + fakeClient({ + connect: async () => { + throw new Error("daemon absent"); + }, + }), + fakeClient({ callToolThrows: true }), + fakeClient({ + connect: async () => { + throw new Error("daemon absent"); + }, + }), + fakeClient({}), + ]; + const spawned: Array<{ signalCode: string | null; exitCode: number | null }> = []; + const spawnProcess = vi.fn(() => { + const child = { + exitCode: null, + signalCode: null, + once: vi.fn(), + unref: vi.fn(), + kill: vi.fn(), + }; + spawned.push(child); + return child as unknown as ChildProcess; + }); + const driver = new CuaDriverClient({ + driverPath: "/opt/bin/cua-driver", + env: { PATH: "/opt/bin" }, + access: () => {}, + clientFactory: () => clients.shift() ?? fakeClient({}), + transportFactory: fakeTransport, + spawn: spawnProcess as unknown as typeof import("node:child_process").spawn, + sleep: async () => {}, + }); + + await expect(driver.callTool("get_screen_size", {})).rejects.toThrow("transport closed"); + const firstChild = spawned[0]; + expect(firstChild).toBeDefined(); + firstChild!.signalCode = "SIGKILL"; + await expect(driver.callTool("get_desktop_state", {})).resolves.toBeDefined(); + expect(spawnProcess).toHaveBeenCalledTimes(2); + }); + + it("respawns after the spawned child emits an async spawn error", async () => { + // `error` can fire without `exit`, leaving exitCode/signalCode null; the + // handler must still forget the child so the next connect respawns. + const clients = [ + fakeClient({ + connect: async () => { + throw new Error("daemon absent"); + }, + }), + fakeClient({ callToolThrows: true }), + fakeClient({ + connect: async () => { + throw new Error("daemon absent"); + }, + }), + fakeClient({}), + ]; + const errorHandlers: Array<() => void> = []; + const spawnProcess = vi.fn(() => { + const child = { + exitCode: null, + signalCode: null, + unref: vi.fn(), + kill: vi.fn(), + once: (event: string, cb: () => void) => { + if (event === "error") { + errorHandlers.push(cb); + } + }, + }; + return child as unknown as ChildProcess; + }); + const driver = new CuaDriverClient({ + driverPath: "/opt/bin/cua-driver", + env: { PATH: "/opt/bin" }, + access: () => {}, + clientFactory: () => clients.shift() ?? fakeClient({}), + transportFactory: fakeTransport, + spawn: spawnProcess as unknown as typeof import("node:child_process").spawn, + sleep: async () => {}, + }); + + await expect(driver.callTool("get_screen_size", {})).rejects.toThrow("transport closed"); + errorHandlers[0]?.(); + await expect(driver.callTool("get_desktop_state", {})).resolves.toBeDefined(); + expect(spawnProcess).toHaveBeenCalledTimes(2); + }); + + it("connects to a shared daemon even when our serve child lost the startup race", async () => { + const clients = [ + fakeClient({ + connect: async () => { + throw new Error("daemon absent"); + }, + }), + fakeClient({}), + ]; + const child = { + exitCode: null, + once: vi.fn(), + unref: vi.fn(), + kill: vi.fn(), + } as unknown as ChildProcess; + const driver = new CuaDriverClient({ + driverPath: "/opt/bin/cua-driver", + env: { PATH: "/opt/bin" }, + access: () => {}, + clientFactory: () => clients.shift() ?? fakeClient({}), + transportFactory: fakeTransport, + spawn: vi.fn(() => child) as unknown as typeof import("node:child_process").spawn, + // Our serve child exited (another client won the race), but the shared + // daemon it collided with is now serving; the retry must still connect. + sleep: async () => { + (child as { exitCode: number | null }).exitCode = 1; + }, + }); + + await expect(driver.callTool("get_screen_size", {})).resolves.toBeDefined(); + }); + + it("gives up with a readiness timeout after exhausting the backoff schedule", async () => { + const delays: number[] = []; + const child = { + exitCode: null, + once: vi.fn(), + unref: vi.fn(), + kill: vi.fn(), + } as unknown as ChildProcess; + const driver = new CuaDriverClient({ + driverPath: "/opt/bin/cua-driver", + env: { PATH: "/opt/bin" }, + access: () => {}, + clientFactory: () => + fakeClient({ + connect: async () => { + throw new Error("daemon never ready"); + }, + }), + transportFactory: fakeTransport, + spawn: vi.fn(() => child) as unknown as typeof import("node:child_process").spawn, + sleep: async (durationMs) => { + delays.push(durationMs); + }, + }); + + await expect(driver.callTool("get_screen_size", {})).rejects.toThrow( + /COMPUTER_DRIVER_UNAVAILABLE: cua-driver daemon did not become ready in time/, + ); + expect(delays).toEqual([250, 500, 1_000, 2_000, 3_000, 3_000]); + }); + + it("closes a connection that completes after disposal starts", async () => { + let finishConnect = () => {}; + const connecting = new Promise((resolve) => { + finishConnect = resolve; + }); + const client = fakeClient({ connect: async () => await connecting }); + const driver = createClient(client); + + const call = driver.callTool("get_screen_size", {}); + const disposal = driver.dispose(); + finishConnect(); + + await expect(call).rejects.toThrow("cua-driver client is disposed"); + await disposal; + expect(client.callTool).not.toHaveBeenCalled(); + expect(client.close).toHaveBeenCalled(); + expect(driver.isAvailable()).toBe(false); + }); + + it("caches binary resolution for one second", () => { + let now = 0; + const access = vi.fn(() => {}); + const driver = new CuaDriverClient({ + env: { PATH: "/one:/two" }, + now: () => now, + access, + }); + expect(driver.isAvailable()).toBe(true); + expect(driver.isAvailable()).toBe(true); + expect(access).toHaveBeenCalledTimes(1); + now = 1_001; + expect(driver.isAvailable()).toBe(true); + expect(access).toHaveBeenCalledTimes(2); + }); + + it("reports unavailable when the binary cannot be resolved", () => { + const driver = new CuaDriverClient({ + env: { PATH: "/missing" }, + access: () => { + throw new Error("ENOENT"); + }, + }); + expect(driver.isAvailable()).toBe(false); + }); +}); + +describe("CuaDriverClient refusal mapping", () => { + it("uses a structured refusal code and the first text block", async () => { + const driver = createClient( + fakeClient({ + result: { + isError: true, + content: [ + { type: "text", text: "desktop unavailable" }, + { type: "text", text: "ignored" }, + ], + structuredContent: { code: "background_unavailable" }, + }, + }), + ); + await expect(driver.callTool("click", {})).rejects.toThrow( + "COMPUTER_REFUSED_background_unavailable: desktop unavailable", + ); + }); + + it("falls back to the generic driver error prefix", async () => { + const driver = createClient( + fakeClient({ + result: { isError: true, content: [{ type: "text", text: "bad input" }] }, + }), + ); + await expect(driver.callTool("click", {})).rejects.toThrow("COMPUTER_DRIVER_ERROR: bad input"); + }); +}); diff --git a/extensions/cua-computer/src/driver-client.ts b/extensions/cua-computer/src/driver-client.ts new file mode 100644 index 000000000000..62e03b286617 --- /dev/null +++ b/extensions/cua-computer/src/driver-client.ts @@ -0,0 +1,514 @@ +import { spawn, type ChildProcess } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { + StdioClientTransport, + type StdioServerParameters, +} from "@modelcontextprotocol/sdk/client/stdio.js"; +import type { Transport } from "@modelcontextprotocol/sdk/shared/transport.js"; + +// cua-driver is prerelease upstream; pin the exact minor contract until it stabilizes. +const SUPPORTED_DRIVER_VERSION_PREFIX = "0.10."; +const BINARY_CACHE_MS = 1_000; +// Cumulative ~9.75s of daemon readiness polling after spawning `serve`. +const DAEMON_READY_BACKOFF_MS = [250, 500, 1_000, 2_000, 3_000, 3_000] as const; +// How long an unsupported-version verdict suppresses re-probes. Bounded so that +// installing the right driver or restarting an incompatible daemon recovers +// without a node restart, while a persistently-wrong driver is not re-probed on +// every call. +const UNSUPPORTED_REPROBE_MS = 30_000; + +type CuaToolContent = + | { type: "text"; text: string } + | { type: "image"; data: string; mimeType: string } + | Record; + +export type CuaToolResult = { + content: CuaToolContent[]; + isError?: boolean; + structuredContent?: Record; +}; + +export interface CuaDriver { + readonly generation: number; + isAvailable(): boolean; + resetAvailabilityCache(): void; + callTool(name: string, args: Record): Promise; + dispose(): Promise; +} + +type McpClientLike = { + connect(transport: Transport): Promise; + getServerVersion(): { name: string; version: string } | undefined; + listTools(): Promise; + callTool(params: { name: string; arguments?: Record }): Promise; + close(): Promise; +}; + +type CuaDriverClientOptions = { + driverPath?: string; + env?: NodeJS.ProcessEnv; + platform?: NodeJS.Platform; + now?: () => number; + access?: (filePath: string, mode: number) => void; + spawn?: typeof spawn; + transportFactory?: (params: StdioServerParameters) => Transport; + clientFactory?: () => McpClientLike; + sleep?: (durationMs: number) => Promise; +}; + +type DriverSession = { + client: McpClientLike; + transport: Transport; +}; + +class ComputerDriverUnsupportedError extends Error { + readonly code = "COMPUTER_DRIVER_UNSUPPORTED"; + + constructor(found: string, pinned: string) { + super(`COMPUTER_DRIVER_UNSUPPORTED: found ${found}; required ${pinned}`); + this.name = "ComputerDriverUnsupportedError"; + } +} + +// cua-driver is a separately installed process that outlives this client, so it +// must never inherit OpenClaw secrets (provider tokens, channel credentials). +// Forward a deny-by-default allowlist of only the OS/session variables the +// driver needs plus its own CUA_/XDG_/LC_ namespaces. +const DRIVER_ENV_ALLOWLIST = new Set( + [ + "PATH", + "HOME", + "USER", + "LOGNAME", + "USERNAME", + "USERDOMAIN", + "LANG", + "LANGUAGE", + "TERM", + "TZ", + "SHELL", + "TMPDIR", + "TEMP", + "TMP", + // Linux X11/Wayland session + "DISPLAY", + "WAYLAND_DISPLAY", + "XAUTHORITY", + "DBUS_SESSION_BUS_ADDRESS", + // Windows system paths the driver's runtime relies on + "USERPROFILE", + "HOMEDRIVE", + "HOMEPATH", + "APPDATA", + "LOCALAPPDATA", + "PROGRAMDATA", + "PROGRAMFILES", + "PROGRAMFILES(X86)", + "COMMONPROGRAMFILES", + "COMMONPROGRAMFILES(X86)", + "SYSTEMROOT", + "SYSTEMDRIVE", + "WINDIR", + "COMSPEC", + "PATHEXT", + "PROCESSOR_ARCHITECTURE", + "NUMBER_OF_PROCESSORS", + "COMPUTERNAME", + "SESSIONNAME", + // cua-driver local config — an explicit list, not a CUA_ prefix, because the + // CUA_ namespace also holds cloud credentials like CUA_API_KEY that this + // local desktop driver never needs. + "CUA_DRIVER_RS_ENABLE_WAYLAND", + "CUA_DRIVER_RS_SESSION_IDLE_TTL_SECS", + "CUA_DRIVER_POLICY_FILE", + "CUA_DRIVER_MANAGED_POLICY_FILE", + "CUA_DRIVER_SESSION_POLICY_FILE", + ].map((name) => name.toUpperCase()), +); + +// Locale and freedesktop session-dir namespaces only. Both are credential-free +// by spec; the CUA_ namespace is deliberately excluded (see the allowlist). +const DRIVER_ENV_ALLOW_PREFIXES = ["XDG_", "LC_"]; + +function buildDriverEnvironment(env: NodeJS.ProcessEnv): Record { + const result: Record = {}; + for (const [key, value] of Object.entries(env)) { + if (typeof value !== "string") { + continue; + } + const upper = key.toUpperCase(); + const allowed = + DRIVER_ENV_ALLOWLIST.has(upper) || + DRIVER_ENV_ALLOW_PREFIXES.some((prefix) => upper.startsWith(prefix)); + if (allowed) { + result[key] = value; + } + } + // Force OpenClaw-managed opt-outs even over an inherited CUA_* value. + result.CUA_DRIVER_RS_TELEMETRY_ENABLED = "false"; + result.CUA_DRIVER_RS_UPDATE_CHECK = "false"; + return result; +} + +function firstTextBlock(content: CuaToolContent[]): string { + const block = content.find( + (entry): entry is { type: "text"; text: string } => + entry.type === "text" && typeof entry.text === "string", + ); + return block?.text ?? "cua-driver tool failed"; +} + +function asRecord(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : {}; +} + +function asToolResult(value: unknown): CuaToolResult { + const record = asRecord(value); + return { + content: Array.isArray(record.content) ? (record.content as CuaToolContent[]) : [], + isError: record.isError === true, + structuredContent: + record.structuredContent && typeof record.structuredContent === "object" + ? (record.structuredContent as Record) + : undefined, + }; +} + +export class CuaDriverClient implements CuaDriver { + private readonly driverPath?: string; + private readonly env: NodeJS.ProcessEnv; + private readonly platform: NodeJS.Platform; + private readonly now: () => number; + private readonly access: (filePath: string, mode: number) => void; + private readonly spawnProcess: typeof spawn; + private readonly transportFactory: (params: StdioServerParameters) => Transport; + private readonly clientFactory: () => McpClientLike; + private readonly sleep: (durationMs: number) => Promise; + private binaryCache: { checkedAt: number; path: string | null } = { + checkedAt: Number.NEGATIVE_INFINITY, + path: null, + }; + private session?: DriverSession; + private connectPromise?: Promise; + private serveProcess?: ChildProcess; + private unsupportedError?: ComputerDriverUnsupportedError; + private unsupportedAt = 0; + private generationValue = 0; + private disposed = false; + + constructor(options: CuaDriverClientOptions = {}) { + this.driverPath = options.driverPath; + this.env = options.env ?? process.env; + this.platform = options.platform ?? process.platform; + this.now = options.now ?? Date.now; + this.access = options.access ?? fs.accessSync; + this.spawnProcess = options.spawn ?? spawn; + this.transportFactory = + options.transportFactory ?? ((params) => new StdioClientTransport(params)); + this.clientFactory = + options.clientFactory ?? + (() => new Client({ name: "openclaw-cua-computer", version: "0.0.0" })); + this.sleep = + options.sleep ?? + (async (durationMs) => { + await new Promise((resolve) => { + const timer = setTimeout(resolve, durationMs); + timer.unref?.(); + }); + }); + } + + get generation(): number { + return this.generationValue; + } + + private executableNames(name: string): string[] { + if (this.platform !== "win32" || path.extname(name)) { + return [name]; + } + const extensions = (this.env.PATHEXT ?? ".EXE;.CMD;.BAT;.COM").split(";").filter(Boolean); + return [name, ...extensions.map((extension) => `${name}${extension.toLowerCase()}`)]; + } + + private canExecute(candidate: string): boolean { + try { + this.access(candidate, fs.constants.X_OK); + return true; + } catch { + return false; + } + } + + private resolveBinaryUncached(): string | null { + const requested = this.driverPath ?? "cua-driver"; + if (path.isAbsolute(requested)) { + return this.canExecute(requested) ? requested : null; + } + const pathEntries = (this.env.PATH ?? "").split(path.delimiter).filter(Boolean); + for (const entry of pathEntries) { + for (const name of this.executableNames(requested)) { + const candidate = path.resolve(entry, name); + if (this.canExecute(candidate)) { + return candidate; + } + } + } + return null; + } + + private resolveBinary(): string | null { + const now = this.now(); + if (now - this.binaryCache.checkedAt < BINARY_CACHE_MS) { + return this.binaryCache.path; + } + const resolved = this.resolveBinaryUncached(); + this.binaryCache = { checkedAt: now, path: resolved }; + return resolved; + } + + /** The cached version-incompatibility error while its re-probe window holds. */ + private activeUnsupportedError(): ComputerDriverUnsupportedError | undefined { + if ( + this.unsupportedError !== undefined && + this.now() - this.unsupportedAt < UNSUPPORTED_REPROBE_MS + ) { + return this.unsupportedError; + } + return undefined; + } + + isAvailable(): boolean { + return ( + !this.disposed && this.activeUnsupportedError() === undefined && this.resolveBinary() !== null + ); + } + + resetAvailabilityCache(): void { + this.binaryCache.checkedAt = Number.NEGATIVE_INFINITY; + } + + private driverEnv(): Record { + return buildDriverEnvironment(this.env); + } + + private async closeSession(session: DriverSession | undefined): Promise { + if (!session) { + return; + } + await session.client.close().catch(() => {}); + await session.transport.close().catch(() => {}); + } + + private async connectOnce(binary: string): Promise { + const transport = this.transportFactory({ + command: binary, + args: ["mcp"], + env: this.driverEnv(), + stderr: "ignore", + }); + const client = this.clientFactory(); + const session = { client, transport }; + try { + await client.connect(transport); + const serverInfo = client.getServerVersion(); + const foundServer = serverInfo + ? `${serverInfo.name}@${serverInfo.version}` + : "missing serverInfo"; + if ( + serverInfo?.name !== "cua-driver" || + !serverInfo.version.startsWith(SUPPORTED_DRIVER_VERSION_PREFIX) + ) { + throw new ComputerDriverUnsupportedError( + foundServer, + `cua-driver@${SUPPORTED_DRIVER_VERSION_PREFIX}x`, + ); + } + const listed = asRecord(await client.listTools()); + const capabilityVersion = listed.capability_version; + const schemaVersion = listed.schema_version; + if (capabilityVersion !== "1" || schemaVersion !== "1") { + throw new ComputerDriverUnsupportedError( + `cua-driver@${serverInfo.version} capability_version=${String(capabilityVersion)} schema_version=${String(schemaVersion)}`, + `cua-driver@${SUPPORTED_DRIVER_VERSION_PREFIX}x capability_version=1 schema_version=1`, + ); + } + this.generationValue += 1; + return session; + } catch (error) { + await this.closeSession(session); + throw error; + } + } + + private spawnDaemon(binary: string): void { + // A signal-terminated child leaves exitCode null but sets signalCode, so + // both must be null to treat the remembered daemon as still running; + // otherwise a SIGKILL/OOM'd daemon would block every future respawn. + if ( + this.serveProcess && + this.serveProcess.exitCode === null && + this.serveProcess.signalCode == null + ) { + return; + } + const child = this.spawnProcess(binary, ["serve"], { + detached: true, + env: this.driverEnv(), + stdio: "ignore", + windowsHide: true, + }); + const forget = () => { + if (this.serveProcess === child) { + this.serveProcess = undefined; + } + }; + // A binary can disappear between the availability check and spawn. `error` + // can fire without `exit`, leaving exitCode/signalCode both null, so forget + // the child here too or the guard above would treat the failed spawn as a + // live daemon forever. The MCP retry owns the actionable failure. + child.once("error", forget); + // Forget the child once it dies (either code or signal) so the next connect + // spawns a fresh daemon instead of trusting a stale handle. + child.once("exit", forget); + child.unref(); + this.serveProcess = child; + } + + private async connect(): Promise { + if (this.disposed) { + throw new Error("COMPUTER_DRIVER_UNAVAILABLE: cua-driver client is disposed"); + } + if (this.session) { + return this.session; + } + const activeUnsupported = this.activeUnsupportedError(); + if (activeUnsupported) { + throw activeUnsupported; + } + // Verdict expired: allow one fresh compatibility probe so a corrected driver + // or restarted daemon recovers without a node restart. + this.unsupportedError = undefined; + if (this.connectPromise) { + return await this.connectPromise; + } + const binary = this.resolveBinary(); + if (!binary) { + throw new Error("COMPUTER_DRIVER_UNAVAILABLE: cua-driver executable not found"); + } + const pending = (async () => { + try { + return await this.connectOnce(binary); + } catch (error) { + if (error instanceof ComputerDriverUnsupportedError) { + this.unsupportedError = error; + this.unsupportedAt = this.now(); + throw error; + } + if (this.disposed) { + throw new Error("COMPUTER_DRIVER_UNAVAILABLE: cua-driver client is disposed", { + cause: error, + }); + } + this.spawnDaemon(binary); + // A cold `serve` start (Xvfb, portals, UIA warmup) can take seconds; + // upstream's own mcp launcher waits up to 10s for the macOS daemon. + // Poll with backoff instead of racing one fixed delay. + const lastIndex = DAEMON_READY_BACKOFF_MS.length - 1; + for (const [index, delayMs] of DAEMON_READY_BACKOFF_MS.entries()) { + await this.sleep(delayMs); + if (this.disposed) { + throw new Error("COMPUTER_DRIVER_UNAVAILABLE: cua-driver client is disposed", { + cause: error, + }); + } + try { + return await this.connectOnce(binary); + } catch (retryError) { + if (retryError instanceof ComputerDriverUnsupportedError) { + this.unsupportedError = retryError; + this.unsupportedAt = this.now(); + throw retryError; + } + // Give up only after the budget is exhausted, reporting the final + // retry failure (the most relevant cause) rather than the first. + // A child exit mid-budget is not terminal: cua-driver allows one + // daemon per endpoint, so ours may have collided with a shared one + // that needs more time to answer. + if (index === lastIndex) { + throw new Error( + "COMPUTER_DRIVER_UNAVAILABLE: cua-driver daemon did not become ready in time", + { cause: retryError }, + ); + } + } + } + // Unreachable: the final iteration always returns or throws. Present for + // control-flow completeness only. + throw new Error("COMPUTER_DRIVER_UNAVAILABLE: cua-driver daemon did not become ready", { + cause: error, + }); + } + })(); + this.connectPromise = pending; + try { + const session = await pending; + if (this.disposed) { + await this.closeSession(session); + throw new Error("COMPUTER_DRIVER_UNAVAILABLE: cua-driver client is disposed"); + } + this.session = session; + return this.session; + } finally { + this.connectPromise = undefined; + } + } + + async callTool(name: string, args: Record): Promise { + const session = await this.connect(); + if (this.disposed) { + throw new Error("COMPUTER_DRIVER_UNAVAILABLE: cua-driver client is disposed"); + } + let result: CuaToolResult; + try { + result = asToolResult(await session.client.callTool({ name, arguments: args })); + } catch (error) { + if (this.session === session) { + this.session = undefined; + } + await this.closeSession(session); + throw error; + } + if (!result.isError) { + return result; + } + const text = firstTextBlock(result.content); + const code = result.structuredContent?.code; + if (typeof code === "string") { + throw new Error(`COMPUTER_REFUSED_${code}: ${text}`); + } + throw new Error(`COMPUTER_DRIVER_ERROR: ${text}`); + } + + async dispose(): Promise { + this.disposed = true; + const session = this.session; + const pending = this.connectPromise; + this.session = undefined; + await this.closeSession(session); + if (pending) { + const pendingSession = await pending.catch(() => undefined); + if (pendingSession && pendingSession !== session) { + await this.closeSession(pendingSession); + } + } + // Do not kill the daemon: cua-driver runs one shared machine daemon per + // endpoint that other clients may attach to, and its idle-session TTL owns + // cleanup. Closing our mcp client already releases our transport session + // upstream. Killing it would disconnect unrelated clients. + this.serveProcess = undefined; + } +} diff --git a/extensions/cua-computer/src/frame.ts b/extensions/cua-computer/src/frame.ts new file mode 100644 index 000000000000..484cc148c8d4 --- /dev/null +++ b/extensions/cua-computer/src/frame.ts @@ -0,0 +1,114 @@ +import { createHash } from "node:crypto"; + +export type CuaDesktopGeometry = { + platform: string; + display: string; + screenWidth: number; + screenHeight: number; + scaleFactor: number; + screenshotWidth: number; + screenshotHeight: number; +}; + +export type CuaScreenSize = { + width: number; + height: number; + scaleFactor: number; +}; + +export type CuaLastFrame = { + id: string; + nativeWidth: number; + nativeHeight: number; + deliveredWidth: number; + deliveredHeight: number; + geometry: CuaScreenSize; +}; + +export type CuaFrameState = { + generation: number; + lastFrame?: CuaLastFrame; +}; + +function staleFrame(message: string): Error { + return new Error(`COMPUTER_STALE_FRAME: ${message}; take a new screenshot`); +} + +/** + * cua-driver exposes only the primary-display label, not a stable display ID. + * Bind authorization to connection generation plus the complete live geometry. + */ +export function issueFrame( + state: CuaFrameState, + geometry: CuaDesktopGeometry, + delivered: { width: number; height: number }, +): string { + const digest = createHash("sha256") + .update( + JSON.stringify([ + state.generation, + geometry.platform, + geometry.display, + geometry.screenWidth, + geometry.screenHeight, + geometry.scaleFactor, + geometry.screenshotWidth, + geometry.screenshotHeight, + ]), + ) + .digest("hex"); + const id = `cua:v1:${digest}`; + state.lastFrame = { + id, + nativeWidth: geometry.screenshotWidth, + nativeHeight: geometry.screenshotHeight, + deliveredWidth: delivered.width, + deliveredHeight: delivered.height, + geometry: { + width: geometry.screenWidth, + height: geometry.screenHeight, + scaleFactor: geometry.scaleFactor, + }, + }; + return id; +} + +// Accepted limitation: cua-driver 0.10 exposes no stable display identity, only +// "display":"primary" (see get_desktop_state). Verification therefore binds to +// connection generation plus full live geometry. The generation counter +// invalidates every frame on any daemon/session reconnect, which covers RDP +// drops and topology changes that break the MCP transport. The only uncaught +// case is a same-geometry primary-display substitution that leaves the +// connection intact — a corner case upstream gives us no signal to detect. +export function verifyFrame( + state: CuaFrameState, + echoedId: string | undefined, + currentScreenSize: CuaScreenSize, +): CuaLastFrame { + const frame = state.lastFrame; + if (!frame || !echoedId || echoedId !== frame.id) { + state.lastFrame = undefined; + throw staleFrame("the coordinate frame is missing or no longer current"); + } + const geometryMatches = + currentScreenSize.width === frame.geometry.width && + currentScreenSize.height === frame.geometry.height && + currentScreenSize.scaleFactor === frame.geometry.scaleFactor; + if (!geometryMatches) { + state.lastFrame = undefined; + throw staleFrame("the primary display geometry changed"); + } + return frame; +} + +export function verifyReferenceWidth( + state: CuaFrameState, + frame: CuaLastFrame, + refWidth: number | undefined, +): void { + if (refWidth === frame.deliveredWidth) { + return; + } + state.lastFrame = undefined; + throw staleFrame("the coordinate reference width changed"); +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c8ccdcd212df..b4369da3e9d8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -647,6 +647,22 @@ importers: specifier: workspace:* version: link:../../packages/plugin-sdk + extensions/cua-computer: + dependencies: + '@modelcontextprotocol/sdk': + specifier: 1.29.0 + version: 1.29.0(zod@4.4.3) + rastermill: + specifier: 0.3.1 + version: 0.3.1 + zod: + specifier: 4.4.3 + version: 4.4.3 + devDependencies: + '@openclaw/plugin-sdk': + specifier: workspace:* + version: link:../../packages/plugin-sdk + extensions/deepgram: devDependencies: '@openclaw/plugin-sdk':