mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
feat(cua-computer): add experimental Windows/Linux computer-use fulfiller (#112267)
* feat(cua-computer): add experimental Windows/Linux computer-use fulfiller Bundled plugin that fulfills the capability-based computer.act + screen.snapshot node contract on Windows and Linux by supervising a pinned cua-driver 0.10.x daemon over MCP stdio. macOS keeps the Peekaboo fulfiller; this plugin is disabled by default and never available on darwin. Grounded in cua-driver 0.10.0 source (tool schemas, refusal codes, coordinate spaces, session/daemon lifecycle). Notable safety and correctness properties: - Deny-by-default env allowlist so OpenClaw secrets (provider/channel tokens, CUA_API_KEY) never reach the separately installed daemon; telemetry and update checks forced off. - Version-gated handshake (exact-minor pin + capability/schema version), time-bounded so a corrected driver recovers without a node restart. - Robust daemon supervision: full readiness-budget polling, startup-race tolerance, signal-death and spawn-error recovery, shared-daemon lifecycle (never killed on dispose). - Frame authorization preserved within upstream limits (generation + full live geometry; capture refused when screen and screenshot geometry diverge). - Action mapping refuses inputs cua-driver cannot faithfully deliver: layout-shifted keys, modifier-held drag/scroll, Linux modifier clicks, hold_key/mouse down-up, non-positive scroll; drag duration clamped. * fix(cua-computer): satisfy lint, test-types, dead-code, and docs-map gates
This commit is contained in:
Peter Steinberger
committed by
GitHub
GitHub
parent
a370785bab
commit
f695be341c
@@ -0,0 +1,21 @@
|
||||
import type {
|
||||
OpenClawPluginApi,
|
||||
OpenClawPluginNodeHostCommand,
|
||||
} from "openclaw/plugin-sdk/plugin-entry";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import plugin from "./index.js";
|
||||
|
||||
describe("cua-computer plugin registration", () => {
|
||||
it("registers the screen and dangerous computer node-host commands", () => {
|
||||
const commands: OpenClawPluginNodeHostCommand[] = [];
|
||||
plugin.register({
|
||||
pluginConfig: { driverPath: "cua-driver" },
|
||||
registerNodeHostCommand: (command: OpenClawPluginNodeHostCommand) => commands.push(command),
|
||||
} as unknown as OpenClawPluginApi);
|
||||
|
||||
expect(commands.map(({ command, cap, dangerous }) => ({ command, cap, dangerous }))).toEqual([
|
||||
{ command: "screen.snapshot", cap: "screen", dangerous: false },
|
||||
{ command: "computer.act", cap: "computer", dangerous: true },
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,34 @@
|
||||
import { buildPluginConfigSchema, definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
|
||||
import { z } from "zod";
|
||||
import { createCuaComputerCommands } from "./src/commands.js";
|
||||
|
||||
const CuaComputerConfigSchema = z.strictObject({
|
||||
driverPath: z.string().trim().min(1).optional(),
|
||||
});
|
||||
|
||||
const configSchema = buildPluginConfigSchema(CuaComputerConfigSchema, {
|
||||
uiHints: {
|
||||
driverPath: {
|
||||
label: "cua-driver path",
|
||||
help: "Absolute path or executable name resolved through PATH. Defaults to cua-driver.",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
export default definePluginEntry({
|
||||
id: "cua-computer",
|
||||
name: "CUA Computer",
|
||||
description: "Experimental cua-driver computer control for Windows and Linux node hosts.",
|
||||
configSchema,
|
||||
register(api) {
|
||||
const parsed = CuaComputerConfigSchema.safeParse(api.pluginConfig ?? {});
|
||||
if (!parsed.success) {
|
||||
throw new Error(
|
||||
`Invalid cua-computer plugin config: ${parsed.error.issues[0]?.message ?? "invalid config"}`,
|
||||
);
|
||||
}
|
||||
for (const command of createCuaComputerCommands({ driverPath: parsed.data.driverPath })) {
|
||||
api.registerNodeHostCommand(command);
|
||||
}
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"id": "cua-computer",
|
||||
"activation": {
|
||||
"onStartup": true
|
||||
},
|
||||
"enabledByDefault": false,
|
||||
"name": "CUA Computer",
|
||||
"description": "Experimental cua-driver computer control for Windows and Linux node hosts.",
|
||||
"configSchema": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"driverPath": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"name": "@openclaw/cua-computer",
|
||||
"version": "2026.7.2",
|
||||
"description": "Experimental cua-driver computer control for Windows and Linux node hosts",
|
||||
"type": "module",
|
||||
"dependencies": {
|
||||
"@modelcontextprotocol/sdk": "1.29.0",
|
||||
"rastermill": "0.3.1",
|
||||
"zod": "4.4.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@openclaw/plugin-sdk": "workspace:*"
|
||||
},
|
||||
"openclaw": {
|
||||
"extensions": [
|
||||
"./index.ts"
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { normalizeModifiers, parseKeyChord, scalePoint } from "./actions.js";
|
||||
|
||||
// normalizeKey is internal; exercise it through parseKeyChord's final segment.
|
||||
const normalizeKey = (input: string) => parseKeyChord(input).key;
|
||||
|
||||
describe("cua-computer key normalization", () => {
|
||||
it.each([
|
||||
["cmd+shift", ["meta", "shift"]],
|
||||
["Super+Control+Option", ["meta", "ctrl", "alt"]],
|
||||
["win+mod1", ["meta", "alt"]],
|
||||
])("normalizes modifier aliases in %s", (input, expected) => {
|
||||
expect(normalizeModifiers(input)).toEqual(expected);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["Return", "enter"],
|
||||
["Esc", "escape"],
|
||||
["PgDn", "pagedown"],
|
||||
["Home", "home"],
|
||||
["F12", "f12"],
|
||||
["z", "z"],
|
||||
["Z", "z"],
|
||||
["c", "c"],
|
||||
])("normalizes key %s", (input, expected) => {
|
||||
expect(normalizeKey(input)).toBe(expected);
|
||||
});
|
||||
|
||||
it.each(["minus", "slash", "equal", "period", "comma", "semicolon"])(
|
||||
"rejects punctuation-alias key %s toward the type action",
|
||||
(input) => {
|
||||
expect(() => normalizeKey(input)).toThrow("COMPUTER_UNSUPPORTED_KEY");
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps letter keys usable in shortcut chords", () => {
|
||||
expect(parseKeyChord("cmd+c")).toEqual({ key: "c", modifiers: ["meta"] });
|
||||
});
|
||||
|
||||
// Digits and punctuation are shifted on some keyboard layouts, and cua-driver
|
||||
// drops that shift state, so they must be rejected toward the type action
|
||||
// rather than silently degraded.
|
||||
it.each(["1", "+", "*", ":", "_", "(", ".", "?", "é"])(
|
||||
"rejects layout-shifted key %s toward the type action",
|
||||
(input) => {
|
||||
expect(() => normalizeKey(input)).toThrow("COMPUTER_UNSUPPORTED_KEY");
|
||||
},
|
||||
);
|
||||
|
||||
it("splits the last chord segment into the key", () => {
|
||||
expect(parseKeyChord("cmd+ctrl+Return")).toEqual({
|
||||
key: "enter",
|
||||
modifiers: ["meta", "ctrl"],
|
||||
});
|
||||
});
|
||||
|
||||
it.each(["hyper", "ctrl+hyper"])("rejects unknown vocabulary in %s", (input) => {
|
||||
const operation = input.includes("+")
|
||||
? () => parseKeyChord(input)
|
||||
: () => normalizeModifiers(input);
|
||||
expect(operation).toThrow("COMPUTER_UNSUPPORTED_KEY");
|
||||
});
|
||||
|
||||
it("keeps rounded coordinates inside the native primary-display bounds", () => {
|
||||
expect(
|
||||
scalePoint(
|
||||
{
|
||||
id: "frame",
|
||||
nativeWidth: 3840,
|
||||
nativeHeight: 2160,
|
||||
deliveredWidth: 1920,
|
||||
deliveredHeight: 1080,
|
||||
geometry: { width: 3840, height: 2160, scaleFactor: 1 },
|
||||
},
|
||||
1919.9,
|
||||
1079.9,
|
||||
"click",
|
||||
),
|
||||
).toEqual({ x: 3839, y: 2159 });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,176 @@
|
||||
import { z } from "zod";
|
||||
import type { CuaLastFrame } from "./frame.js";
|
||||
|
||||
const COMPUTER_ACTIONS = [
|
||||
"left_click",
|
||||
"right_click",
|
||||
"middle_click",
|
||||
"double_click",
|
||||
"triple_click",
|
||||
"mouse_move",
|
||||
"left_click_drag",
|
||||
"left_mouse_down",
|
||||
"left_mouse_up",
|
||||
"scroll",
|
||||
"type",
|
||||
"key",
|
||||
"hold_key",
|
||||
] as const;
|
||||
|
||||
export const ComputerActParamsSchema = z.strictObject({
|
||||
action: z.enum(COMPUTER_ACTIONS),
|
||||
displayFrameId: z.string().optional(),
|
||||
x: z.number().finite().nonnegative().optional(),
|
||||
y: z.number().finite().nonnegative().optional(),
|
||||
fromX: z.number().finite().nonnegative().optional(),
|
||||
fromY: z.number().finite().nonnegative().optional(),
|
||||
text: z.string().optional(),
|
||||
keys: z.string().optional(),
|
||||
modifiers: z.string().optional(),
|
||||
scrollDirection: z.enum(["up", "down", "left", "right"]).optional(),
|
||||
scrollAmount: z.number().int().positive().optional(),
|
||||
durationMs: z.number().int().nonnegative().optional(),
|
||||
screenIndex: z.number().int().nonnegative().optional(),
|
||||
refWidth: z.number().int().positive().optional(),
|
||||
});
|
||||
|
||||
export type ComputerActParams = z.infer<typeof ComputerActParamsSchema>;
|
||||
|
||||
const MODIFIER_ALIASES = new Map<string, string>([
|
||||
["ctrl", "ctrl"],
|
||||
["control", "ctrl"],
|
||||
["shift", "shift"],
|
||||
["alt", "alt"],
|
||||
["menu", "alt"],
|
||||
["option", "alt"],
|
||||
["mod1", "alt"],
|
||||
["cmd", "meta"],
|
||||
["command", "meta"],
|
||||
["meta", "meta"],
|
||||
["super", "meta"],
|
||||
["win", "meta"],
|
||||
["windows", "meta"],
|
||||
["mod4", "meta"],
|
||||
]);
|
||||
|
||||
const KEY_ALIASES = new Map<string, string>([
|
||||
["return", "enter"],
|
||||
["enter", "enter"],
|
||||
["tab", "tab"],
|
||||
["escape", "escape"],
|
||||
["esc", "escape"],
|
||||
["space", "space"],
|
||||
["backspace", "backspace"],
|
||||
["delete", "delete"],
|
||||
["del", "delete"],
|
||||
["insert", "insert"],
|
||||
["ins", "insert"],
|
||||
["home", "home"],
|
||||
["end", "end"],
|
||||
["pageup", "pageup"],
|
||||
["pgup", "pageup"],
|
||||
["pagedown", "pagedown"],
|
||||
["pgdn", "pagedown"],
|
||||
["up", "up"],
|
||||
["down", "down"],
|
||||
["left", "left"],
|
||||
["right", "right"],
|
||||
["capslock", "capslock"],
|
||||
["numlock", "numlock"],
|
||||
// Punctuation aliases are intentionally absent: they resolve to characters
|
||||
// whose shift/AltGr state is layout-dependent and dropped by cua-driver, the
|
||||
// same reason single punctuation chars are rejected below. Route them to the
|
||||
// `type` action instead.
|
||||
]);
|
||||
|
||||
for (let index = 1; index <= 12; index += 1) {
|
||||
KEY_ALIASES.set(`f${index}`, `f${index}`);
|
||||
}
|
||||
|
||||
function unsupportedKey(message: string): Error {
|
||||
return new Error(`COMPUTER_UNSUPPORTED_KEY: ${message}`);
|
||||
}
|
||||
|
||||
export function normalizeModifiers(value: string | undefined): string[] {
|
||||
if (!value?.trim()) {
|
||||
return [];
|
||||
}
|
||||
return value.split("+").map((entry) => {
|
||||
const raw = entry.trim();
|
||||
const normalized = MODIFIER_ALIASES.get(raw.toLowerCase());
|
||||
if (!normalized) {
|
||||
throw unsupportedKey(`unknown modifier ${JSON.stringify(raw)}`);
|
||||
}
|
||||
return normalized;
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeKey(value: string): string {
|
||||
const raw = value.trim();
|
||||
if (!raw) {
|
||||
throw unsupportedKey("key chord contains an empty key");
|
||||
}
|
||||
const lowered = raw.toLowerCase();
|
||||
const modifier = MODIFIER_ALIASES.get(lowered);
|
||||
if (modifier) {
|
||||
return modifier;
|
||||
}
|
||||
const named = KEY_ALIASES.get(lowered);
|
||||
if (named) {
|
||||
return named;
|
||||
}
|
||||
// cua-driver 0.10 resolves single characters through VkKeyScanW/keysym lookups
|
||||
// and keeps only the base virtual key, dropping the shift/AltGr state the
|
||||
// active layout needs (keyboard.rs key_name_to_vk). ASCII letters are unshifted
|
||||
// in every Latin layout, so they stay valid chord keys (e.g. ctrl+c). Digits
|
||||
// and punctuation are shifted on some layouts (AZERTY digits, US symbols), so
|
||||
// they are rejected toward the `type` action rather than mis-sent.
|
||||
if (/^[a-z]$/i.test(raw)) {
|
||||
return lowered;
|
||||
}
|
||||
if (raw.length === 1) {
|
||||
throw unsupportedKey(
|
||||
`single-character key ${JSON.stringify(raw)} loses layout shift state in cua-driver; use the type action instead`,
|
||||
);
|
||||
}
|
||||
throw unsupportedKey(`unknown key ${JSON.stringify(raw)}`);
|
||||
}
|
||||
|
||||
export function parseKeyChord(value: string | undefined): { key: string; modifiers: string[] } {
|
||||
const segments = value?.split("+").map((entry) => entry.trim()) ?? [];
|
||||
const rawKey = segments.pop();
|
||||
if (!rawKey) {
|
||||
throw unsupportedKey("key chord is empty");
|
||||
}
|
||||
const modifiers = segments.map((entry) => {
|
||||
const normalized = MODIFIER_ALIASES.get(entry.toLowerCase());
|
||||
if (!normalized) {
|
||||
throw unsupportedKey(`unknown modifier ${JSON.stringify(entry)}`);
|
||||
}
|
||||
return normalized;
|
||||
});
|
||||
return { key: normalizeKey(rawKey), modifiers };
|
||||
}
|
||||
|
||||
export function scalePoint(
|
||||
frame: CuaLastFrame,
|
||||
x: number | undefined,
|
||||
y: number | undefined,
|
||||
label: string,
|
||||
): { x: number; y: number } {
|
||||
if (x === undefined || y === undefined) {
|
||||
throw new Error(`COMPUTER_INVALID_REQUEST: ${label} coordinates are required`);
|
||||
}
|
||||
if (x >= frame.deliveredWidth || y >= frame.deliveredHeight) {
|
||||
throw new Error(
|
||||
`COMPUTER_INVALID_REQUEST: ${label} coordinates are outside the captured primary-display frame`,
|
||||
);
|
||||
}
|
||||
return {
|
||||
x: Math.min(frame.nativeWidth - 1, Math.round((x * frame.nativeWidth) / frame.deliveredWidth)),
|
||||
y: Math.min(
|
||||
frame.nativeHeight - 1,
|
||||
Math.round((y * frame.nativeHeight) / frame.deliveredHeight),
|
||||
),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,602 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { createCuaComputerCommands } from "./commands.js";
|
||||
import type { CuaDriver, CuaToolResult } from "./driver-client.js";
|
||||
|
||||
type ToolCall = { name: string; args: Record<string, unknown> };
|
||||
|
||||
function desktopResult(overrides: Record<string, unknown> = {}): CuaToolResult {
|
||||
return {
|
||||
content: [
|
||||
{ type: "image", data: Buffer.from("native-png").toString("base64"), mimeType: "image/png" },
|
||||
{ type: "text", text: "desktop" },
|
||||
],
|
||||
structuredContent: {
|
||||
platform: "linux",
|
||||
display: "primary",
|
||||
screenshot_width: 3840,
|
||||
screenshot_height: 2160,
|
||||
screen_width: 3840,
|
||||
screen_height: 2160,
|
||||
scale_factor: 1,
|
||||
...overrides,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function screenSizeResult(overrides: Record<string, unknown> = {}): CuaToolResult {
|
||||
return {
|
||||
content: [{ type: "text", text: "size" }],
|
||||
structuredContent: { width: 3840, height: 2160, scale_factor: 1, ...overrides },
|
||||
};
|
||||
}
|
||||
|
||||
function createDriver(
|
||||
options: {
|
||||
desktop?: () => CuaToolResult;
|
||||
callTool?: (name: string, args: Record<string, unknown>) => Promise<CuaToolResult>;
|
||||
available?: boolean;
|
||||
generation?: () => number;
|
||||
} = {},
|
||||
) {
|
||||
const calls: ToolCall[] = [];
|
||||
const driver: CuaDriver = {
|
||||
get generation() {
|
||||
return options.generation?.() ?? 1;
|
||||
},
|
||||
isAvailable: () => options.available ?? true,
|
||||
resetAvailabilityCache: vi.fn(),
|
||||
callTool: async (name, args) => {
|
||||
calls.push({ name, args });
|
||||
if (options.callTool) {
|
||||
return await options.callTool(name, args);
|
||||
}
|
||||
if (name === "get_desktop_state") {
|
||||
return options.desktop?.() ?? desktopResult();
|
||||
}
|
||||
if (name === "get_screen_size") {
|
||||
return screenSizeResult();
|
||||
}
|
||||
return { content: [{ type: "text", text: "ok" }] };
|
||||
},
|
||||
dispose: vi.fn(async () => {}),
|
||||
};
|
||||
return { driver, calls };
|
||||
}
|
||||
|
||||
function createProcessor() {
|
||||
const encode = vi.fn(
|
||||
async (
|
||||
_input: Buffer,
|
||||
options: { format: "jpeg" | "png"; quality?: number; resize?: { width: number } },
|
||||
) => ({
|
||||
data: Buffer.from(`${options.format}-encoded`),
|
||||
width: options.resize?.width ?? 3840,
|
||||
height: options.resize ? Math.round((2160 * options.resize.width) / 3840) : 2160,
|
||||
}),
|
||||
);
|
||||
return { processor: { encode }, encode };
|
||||
}
|
||||
|
||||
function commandSet(
|
||||
driver: CuaDriver,
|
||||
imageProcessor = createProcessor().processor,
|
||||
platform: NodeJS.Platform = "linux",
|
||||
) {
|
||||
const commands = createCuaComputerCommands({ platform, driver, imageProcessor });
|
||||
const snapshot = commands.find((command) => command.command === "screen.snapshot");
|
||||
const act = commands.find((command) => command.command === "computer.act");
|
||||
if (!snapshot || !act) {
|
||||
throw new Error("commands missing");
|
||||
}
|
||||
return { snapshot, act };
|
||||
}
|
||||
|
||||
async function issueFrameFor(driver: CuaDriver, platform: NodeJS.Platform = "linux") {
|
||||
const { snapshot, act } = commandSet(driver, createProcessor().processor, platform);
|
||||
const payload = JSON.parse(
|
||||
await snapshot.handle(JSON.stringify({ maxWidth: 1920, format: "jpeg" })),
|
||||
) as { displayFrameId: string; width: number };
|
||||
return { act, frameId: payload.displayFrameId, refWidth: payload.width };
|
||||
}
|
||||
|
||||
describe("cua-computer screen.snapshot", () => {
|
||||
it("scales native screenshots to maxWidth and returns delivered dimensions", async () => {
|
||||
const { driver } = createDriver();
|
||||
const { processor, encode } = createProcessor();
|
||||
const { snapshot } = commandSet(driver, processor);
|
||||
|
||||
const payload = JSON.parse(
|
||||
await snapshot.handle(JSON.stringify({ maxWidth: 1456, quality: 0.61, format: "jpeg" })),
|
||||
) as Record<string, unknown>;
|
||||
|
||||
expect(payload).toMatchObject({ format: "jpeg", screenIndex: 0, width: 1456, height: 819 });
|
||||
expect(payload.displayFrameId).toMatch(/^cua:v1:[a-f0-9]{64}$/);
|
||||
expect(encode).toHaveBeenCalledWith(
|
||||
expect.any(Buffer),
|
||||
expect.objectContaining({
|
||||
format: "jpeg",
|
||||
quality: 61,
|
||||
resize: { width: 1456, enlarge: false },
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it.each(["png", "jpeg"] as const)("encodes %s snapshots", async (format) => {
|
||||
const { driver } = createDriver();
|
||||
const { processor, encode } = createProcessor();
|
||||
const { snapshot } = commandSet(driver, processor);
|
||||
|
||||
const payload = JSON.parse(
|
||||
await snapshot.handle(JSON.stringify({ maxWidth: 2000, format })),
|
||||
) as { format: string; base64: string };
|
||||
|
||||
expect(payload.format).toBe(format);
|
||||
expect(Buffer.from(payload.base64, "base64").toString()).toBe(`${format}-encoded`);
|
||||
expect(encode).toHaveBeenCalledWith(expect.any(Buffer), expect.objectContaining({ format }));
|
||||
});
|
||||
|
||||
it("returns the native PNG without re-encoding when no resize is needed", async () => {
|
||||
const { driver } = createDriver();
|
||||
const { processor, encode } = createProcessor();
|
||||
const { snapshot } = commandSet(driver, processor);
|
||||
|
||||
const payload = JSON.parse(
|
||||
await snapshot.handle(JSON.stringify({ maxWidth: 4000, format: "png" })),
|
||||
) as { base64: string; width: number; height: number };
|
||||
|
||||
expect(Buffer.from(payload.base64, "base64").toString()).toBe("native-png");
|
||||
expect(payload).toMatchObject({ width: 3840, height: 2160 });
|
||||
expect(encode).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects non-primary screen indexes", async () => {
|
||||
const { driver } = createDriver();
|
||||
const { snapshot } = commandSet(driver);
|
||||
await expect(snapshot.handle('{"screenIndex":1}')).rejects.toThrow(
|
||||
"COMPUTER_UNSUPPORTED_DISPLAY",
|
||||
);
|
||||
});
|
||||
|
||||
it("refuses capture when screen and screenshot geometry differ", async () => {
|
||||
// Guards the scalePoint invariant: input is native-pixel, so a capture whose
|
||||
// screen geometry differs from its screenshot pixels would mis-target.
|
||||
const { driver } = createDriver({
|
||||
desktop: () => desktopResult({ screen_width: 2560, screen_height: 1440 }),
|
||||
});
|
||||
const { snapshot } = commandSet(driver);
|
||||
await expect(snapshot.handle("{}")).rejects.toThrow(
|
||||
"COMPUTER_UNSUPPORTED_DISPLAY: cua-driver reported capture and screen geometry",
|
||||
);
|
||||
});
|
||||
|
||||
it("rotates the frame token when captured display geometry changes", async () => {
|
||||
let width = 3840;
|
||||
const { driver } = createDriver({
|
||||
desktop: () =>
|
||||
desktopResult({
|
||||
screenshot_width: width,
|
||||
screen_width: width,
|
||||
}),
|
||||
});
|
||||
const { snapshot } = commandSet(driver);
|
||||
const first = JSON.parse(await snapshot.handle('{"format":"png","maxWidth":4000}')) as {
|
||||
displayFrameId: string;
|
||||
};
|
||||
width = 2560;
|
||||
const second = JSON.parse(await snapshot.handle('{"format":"png","maxWidth":4000}')) as {
|
||||
displayFrameId: string;
|
||||
};
|
||||
expect(second.displayFrameId).not.toBe(first.displayFrameId);
|
||||
});
|
||||
});
|
||||
|
||||
describe("cua-computer computer.act", () => {
|
||||
it("maps all supported pointer actions to desktop-scope driver calls", async () => {
|
||||
const { driver, calls } = createDriver();
|
||||
const { act, frameId, refWidth } = await issueFrameFor(driver);
|
||||
calls.length = 0;
|
||||
const base = { displayFrameId: frameId, refWidth, x: 960, y: 540 };
|
||||
const cases = [
|
||||
["left_click", "click", { x: 1920, y: 1080, button: "left", count: 1 }],
|
||||
["right_click", "click", { x: 1920, y: 1080, button: "right", count: 1 }],
|
||||
["middle_click", "click", { x: 1920, y: 1080, button: "middle", count: 1 }],
|
||||
["double_click", "click", { x: 1920, y: 1080, button: "left", count: 2 }],
|
||||
["triple_click", "click", { x: 1920, y: 1080, button: "left", count: 3 }],
|
||||
["mouse_move", "move_cursor", { x: 1920, y: 1080 }],
|
||||
["left_click_drag", "drag", { from_x: 200, from_y: 400, to_x: 1920, to_y: 1080 }],
|
||||
["scroll", "scroll", { x: 1920, y: 1080, direction: "down", amount: 50, by: "line" }],
|
||||
] as const;
|
||||
|
||||
for (const [action, tool, expected] of cases) {
|
||||
await act.handle(
|
||||
JSON.stringify({
|
||||
action,
|
||||
...base,
|
||||
...(action === "left_click_drag" ? { fromX: 100, fromY: 200 } : {}),
|
||||
...(action === "scroll" ? { scrollDirection: "down", scrollAmount: 99 } : {}),
|
||||
}),
|
||||
);
|
||||
const call = calls.at(-1);
|
||||
expect(call?.name).toBe(tool);
|
||||
expect(call?.args).toMatchObject({ ...expected, scope: "desktop" });
|
||||
}
|
||||
});
|
||||
|
||||
it("normalizes click modifiers and keyboard chords", async () => {
|
||||
const { driver, calls } = createDriver();
|
||||
const { act, frameId, refWidth } = await issueFrameFor(driver, "win32");
|
||||
calls.length = 0;
|
||||
|
||||
await act.handle(
|
||||
JSON.stringify({
|
||||
action: "left_click",
|
||||
displayFrameId: frameId,
|
||||
refWidth,
|
||||
x: 10,
|
||||
y: 20,
|
||||
modifiers: "cmd+Control",
|
||||
}),
|
||||
);
|
||||
await act.handle(JSON.stringify({ action: "key", keys: "super+shift+Return" }));
|
||||
|
||||
expect(calls.find((call) => call.name === "click")?.args).toMatchObject({
|
||||
modifier: ["meta", "ctrl"],
|
||||
});
|
||||
expect(calls.find((call) => call.name === "press_key")?.args).toEqual({
|
||||
key: "enter",
|
||||
modifiers: ["meta", "shift"],
|
||||
scope: "desktop",
|
||||
});
|
||||
});
|
||||
|
||||
it("maps type without geometry calls and rejects the wire-less wait action", async () => {
|
||||
const { driver, calls } = createDriver();
|
||||
const { act } = commandSet(driver);
|
||||
|
||||
await act.handle(JSON.stringify({ action: "type", text: "hello" }));
|
||||
// Core sleeps locally for wait and never sends it over the wire; the
|
||||
// fulfiller must reject it so the computer.act contract stays uniform.
|
||||
await expect(act.handle(JSON.stringify({ action: "wait", durationMs: 25 }))).rejects.toThrow(
|
||||
/COMPUTER_INVALID_REQUEST/,
|
||||
);
|
||||
|
||||
expect(calls).toEqual([{ name: "type_text", args: { text: "hello", scope: "desktop" } }]);
|
||||
});
|
||||
|
||||
it("scrolls at frame-authorized coordinates", async () => {
|
||||
const { driver, calls } = createDriver();
|
||||
const { act, frameId, refWidth } = await issueFrameFor(driver);
|
||||
|
||||
await act.handle(
|
||||
JSON.stringify({
|
||||
action: "scroll",
|
||||
scrollDirection: "up",
|
||||
scrollAmount: 2,
|
||||
displayFrameId: frameId,
|
||||
refWidth,
|
||||
x: 960,
|
||||
y: 540,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(calls.at(-1)).toEqual({
|
||||
name: "scroll",
|
||||
args: { direction: "up", amount: 2, by: "line", x: 1920, y: 1080, scope: "desktop" },
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects coordinate-less scroll instead of guessing the cursor point", async () => {
|
||||
const { driver, calls } = createDriver();
|
||||
const { act } = commandSet(driver);
|
||||
|
||||
await expect(
|
||||
act.handle(JSON.stringify({ action: "scroll", scrollDirection: "up", scrollAmount: 2 })),
|
||||
).rejects.toThrow("COMPUTER_STALE_FRAME");
|
||||
expect(calls.some((call) => call.name === "get_cursor_position")).toBe(false);
|
||||
});
|
||||
|
||||
it.each([0, -3])("rejects non-positive scroll amount %s instead of scrolling", async (amount) => {
|
||||
const { driver, calls } = createDriver();
|
||||
const { act, frameId, refWidth } = await issueFrameFor(driver);
|
||||
await expect(
|
||||
act.handle(
|
||||
JSON.stringify({
|
||||
action: "scroll",
|
||||
scrollDirection: "up",
|
||||
scrollAmount: amount,
|
||||
displayFrameId: frameId,
|
||||
refWidth,
|
||||
x: 10,
|
||||
y: 10,
|
||||
}),
|
||||
),
|
||||
).rejects.toThrow("COMPUTER_INVALID_REQUEST");
|
||||
expect(calls.some((call) => call.name === "scroll")).toBe(false);
|
||||
});
|
||||
|
||||
it("performs an unmodified drag with scaled coordinates", async () => {
|
||||
const { driver, calls } = createDriver();
|
||||
const { act, frameId, refWidth } = await issueFrameFor(driver);
|
||||
|
||||
await act.handle(
|
||||
JSON.stringify({
|
||||
action: "left_click_drag",
|
||||
displayFrameId: frameId,
|
||||
refWidth,
|
||||
fromX: 0,
|
||||
fromY: 0,
|
||||
x: 960,
|
||||
y: 540,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(calls.at(-1)).toEqual({
|
||||
name: "drag",
|
||||
args: { from_x: 0, from_y: 0, to_x: 1920, to_y: 1080, scope: "desktop" },
|
||||
});
|
||||
});
|
||||
|
||||
it("clamps drag duration to the driver's supported maximum", async () => {
|
||||
const { driver, calls } = createDriver();
|
||||
const { act, frameId, refWidth } = await issueFrameFor(driver);
|
||||
|
||||
await act.handle(
|
||||
JSON.stringify({
|
||||
action: "left_click_drag",
|
||||
displayFrameId: frameId,
|
||||
refWidth,
|
||||
fromX: 0,
|
||||
fromY: 0,
|
||||
x: 960,
|
||||
y: 540,
|
||||
durationMs: 15_000,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(calls.at(-1)).toEqual({
|
||||
name: "drag",
|
||||
args: { from_x: 0, from_y: 0, to_x: 1920, to_y: 1080, scope: "desktop", duration_ms: 10_000 },
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects modifier-held drags that cua-driver silently drops", async () => {
|
||||
const { driver } = createDriver();
|
||||
const { act, frameId, refWidth } = await issueFrameFor(driver);
|
||||
|
||||
await expect(
|
||||
act.handle(
|
||||
JSON.stringify({
|
||||
action: "left_click_drag",
|
||||
displayFrameId: frameId,
|
||||
refWidth,
|
||||
fromX: 0,
|
||||
fromY: 0,
|
||||
x: 960,
|
||||
y: 540,
|
||||
modifiers: "shift",
|
||||
}),
|
||||
),
|
||||
).rejects.toThrow("modifier-held drag is unsupported");
|
||||
});
|
||||
|
||||
it("rejects modifier actions that cua-driver cannot preserve", async () => {
|
||||
const { driver } = createDriver();
|
||||
const { act, frameId, refWidth } = await issueFrameFor(driver);
|
||||
await expect(
|
||||
act.handle(
|
||||
JSON.stringify({
|
||||
action: "left_click",
|
||||
displayFrameId: frameId,
|
||||
refWidth,
|
||||
x: 1,
|
||||
y: 1,
|
||||
modifiers: "shift",
|
||||
}),
|
||||
),
|
||||
).rejects.toThrow("modifier-held clicks are unsupported");
|
||||
await expect(
|
||||
act.handle(JSON.stringify({ action: "scroll", scrollDirection: "down", modifiers: "ctrl" })),
|
||||
).rejects.toThrow("modifier-held scroll is unsupported");
|
||||
});
|
||||
|
||||
it.each(["hold_key", "left_mouse_down", "left_mouse_up"])(
|
||||
"rejects unsupported %s",
|
||||
async (action) => {
|
||||
const { driver } = createDriver();
|
||||
const { act } = commandSet(driver);
|
||||
await expect(act.handle(JSON.stringify({ action }))).rejects.toThrow(
|
||||
`COMPUTER_UNSUPPORTED_ACTION: ${action}`,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it("rejects wrong ids, geometry drift, and reference-width drift", async () => {
|
||||
let currentSize = screenSizeResult();
|
||||
const { driver } = createDriver({
|
||||
callTool: async (name) => {
|
||||
if (name === "get_desktop_state") {
|
||||
return desktopResult();
|
||||
}
|
||||
if (name === "get_screen_size") {
|
||||
return currentSize;
|
||||
}
|
||||
return { content: [] };
|
||||
},
|
||||
});
|
||||
|
||||
const wrong = await issueFrameFor(driver);
|
||||
await expect(
|
||||
wrong.act.handle(
|
||||
JSON.stringify({
|
||||
action: "left_click",
|
||||
displayFrameId: "cua:v1:wrong",
|
||||
refWidth: wrong.refWidth,
|
||||
x: 1,
|
||||
y: 1,
|
||||
}),
|
||||
),
|
||||
).rejects.toThrow("COMPUTER_STALE_FRAME");
|
||||
|
||||
const drift = await issueFrameFor(driver);
|
||||
currentSize = screenSizeResult({ width: 2560 });
|
||||
await expect(
|
||||
drift.act.handle(
|
||||
JSON.stringify({
|
||||
action: "mouse_move",
|
||||
displayFrameId: drift.frameId,
|
||||
refWidth: drift.refWidth,
|
||||
x: 1,
|
||||
y: 1,
|
||||
}),
|
||||
),
|
||||
).rejects.toThrow("COMPUTER_STALE_FRAME");
|
||||
|
||||
currentSize = screenSizeResult();
|
||||
const width = await issueFrameFor(driver);
|
||||
await expect(
|
||||
width.act.handle(
|
||||
JSON.stringify({
|
||||
action: "mouse_move",
|
||||
displayFrameId: width.frameId,
|
||||
refWidth: width.refWidth + 1,
|
||||
x: 1,
|
||||
y: 1,
|
||||
}),
|
||||
),
|
||||
).rejects.toThrow("COMPUTER_STALE_FRAME");
|
||||
|
||||
const missing = await issueFrameFor(driver);
|
||||
await expect(
|
||||
missing.act.handle(
|
||||
JSON.stringify({
|
||||
action: "mouse_move",
|
||||
displayFrameId: missing.frameId,
|
||||
x: 1,
|
||||
y: 1,
|
||||
}),
|
||||
),
|
||||
).rejects.toThrow("COMPUTER_STALE_FRAME");
|
||||
});
|
||||
|
||||
it("rejects frames across driver reconnects", async () => {
|
||||
let generation = 1;
|
||||
const { driver } = createDriver({
|
||||
generation: () => generation,
|
||||
callTool: async (name) => {
|
||||
if (name === "get_desktop_state") {
|
||||
return desktopResult();
|
||||
}
|
||||
if (name === "get_screen_size") {
|
||||
generation = 2;
|
||||
return screenSizeResult();
|
||||
}
|
||||
return { content: [] };
|
||||
},
|
||||
});
|
||||
const { act, frameId, refWidth } = await issueFrameFor(driver);
|
||||
await expect(
|
||||
act.handle(
|
||||
JSON.stringify({ action: "mouse_move", displayFrameId: frameId, refWidth, x: 1, y: 1 }),
|
||||
),
|
||||
).rejects.toThrow("the computer driver reconnected");
|
||||
});
|
||||
|
||||
it("rejects coordinates outside the delivered primary-display frame", async () => {
|
||||
const { driver } = createDriver();
|
||||
const { act, frameId, refWidth } = await issueFrameFor(driver);
|
||||
await expect(
|
||||
act.handle(
|
||||
JSON.stringify({
|
||||
action: "left_click",
|
||||
displayFrameId: frameId,
|
||||
refWidth,
|
||||
x: refWidth,
|
||||
y: 0,
|
||||
}),
|
||||
),
|
||||
).rejects.toThrow("outside the captured primary-display frame");
|
||||
});
|
||||
|
||||
it("preserves structured driver refusal errors", async () => {
|
||||
const { driver } = createDriver({
|
||||
callTool: async (name) => {
|
||||
if (name === "get_desktop_state") {
|
||||
return desktopResult();
|
||||
}
|
||||
if (name === "get_screen_size") {
|
||||
return screenSizeResult();
|
||||
}
|
||||
throw new Error("COMPUTER_REFUSED_background_unavailable: desktop unavailable");
|
||||
},
|
||||
});
|
||||
const { act, frameId, refWidth } = await issueFrameFor(driver);
|
||||
await expect(
|
||||
act.handle(
|
||||
JSON.stringify({ action: "left_click", displayFrameId: frameId, refWidth, x: 1, y: 1 }),
|
||||
),
|
||||
).rejects.toThrow("COMPUTER_REFUSED_background_unavailable");
|
||||
});
|
||||
|
||||
it("serializes interleaved action calls", async () => {
|
||||
let releaseFirst = () => {};
|
||||
const firstPending = new Promise<void>((resolve) => {
|
||||
releaseFirst = resolve;
|
||||
});
|
||||
const started: string[] = [];
|
||||
const { driver } = createDriver({
|
||||
callTool: async (name, args) => {
|
||||
if (name === "type_text") {
|
||||
started.push(String(args.text));
|
||||
if (args.text === "first") {
|
||||
await firstPending;
|
||||
}
|
||||
}
|
||||
return { content: [] };
|
||||
},
|
||||
});
|
||||
const { act } = commandSet(driver);
|
||||
|
||||
const first = act.handle(JSON.stringify({ action: "type", text: "first" }));
|
||||
const second = act.handle(JSON.stringify({ action: "type", text: "second" }));
|
||||
await vi.waitFor(() => expect(started).toEqual(["first"]));
|
||||
releaseFirst();
|
||||
await Promise.all([first, second]);
|
||||
expect(started).toEqual(["first", "second"]);
|
||||
});
|
||||
|
||||
it("rejects unknown key and modifier names", async () => {
|
||||
const { driver } = createDriver();
|
||||
const { act, frameId, refWidth } = await issueFrameFor(driver);
|
||||
await expect(act.handle(JSON.stringify({ action: "key", keys: "hyper+x" }))).rejects.toThrow(
|
||||
"COMPUTER_UNSUPPORTED_KEY",
|
||||
);
|
||||
await expect(
|
||||
act.handle(
|
||||
JSON.stringify({
|
||||
action: "left_click",
|
||||
displayFrameId: frameId,
|
||||
refWidth,
|
||||
x: 1,
|
||||
y: 1,
|
||||
modifiers: "hyper",
|
||||
}),
|
||||
),
|
||||
).rejects.toThrow("COMPUTER_UNSUPPORTED_KEY");
|
||||
});
|
||||
});
|
||||
|
||||
describe("cua-computer availability", () => {
|
||||
it.each([
|
||||
["darwin", true, false],
|
||||
["linux", true, true],
|
||||
["linux", false, false],
|
||||
] as const)("returns %s availability with binary=%s", (platform, binary, expected) => {
|
||||
const { driver } = createDriver({ available: binary });
|
||||
const command = createCuaComputerCommands({
|
||||
platform,
|
||||
driver,
|
||||
imageProcessor: createProcessor().processor,
|
||||
})[0];
|
||||
expect(command?.isAvailable?.({ config: {}, env: {} })).toBe(expected);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,462 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import type { OpenClawPluginNodeHostCommand } from "openclaw/plugin-sdk/plugin-entry";
|
||||
import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path";
|
||||
import { createRastermill } from "rastermill";
|
||||
import { z } from "zod";
|
||||
import {
|
||||
ComputerActParamsSchema,
|
||||
normalizeModifiers,
|
||||
parseKeyChord,
|
||||
scalePoint,
|
||||
type ComputerActParams,
|
||||
} from "./actions.js";
|
||||
import { CuaDriverClient, type CuaDriver, type CuaToolResult } from "./driver-client.js";
|
||||
import {
|
||||
issueFrame,
|
||||
verifyFrame,
|
||||
verifyReferenceWidth,
|
||||
type CuaDesktopGeometry,
|
||||
type CuaFrameState,
|
||||
type CuaLastFrame,
|
||||
type CuaScreenSize,
|
||||
} from "./frame.js";
|
||||
|
||||
const AVAILABILITY_POLL_MS = 5_000;
|
||||
// Rastermill enforces inputPixels before resizing, so this must clear the native
|
||||
// capture, not the delivered frame. 8K (7680x4320 = ~33.2M) is a valid primary
|
||||
// display; budget above it so full-resolution snapshots reach the downscaler.
|
||||
const MAX_IMAGE_PIXELS = 40_000_000;
|
||||
|
||||
const SnapshotParamsSchema = z.strictObject({
|
||||
screenIndex: z.number().int().nonnegative().optional(),
|
||||
maxWidth: z.number().int().positive().optional(),
|
||||
quality: z.number().finite().optional(),
|
||||
format: z.enum(["jpeg", "png"]).optional(),
|
||||
});
|
||||
|
||||
const DesktopStateSchema = z.object({
|
||||
platform: z.string().min(1),
|
||||
display: z.string().min(1),
|
||||
screenshot_width: z.number().int().positive(),
|
||||
screenshot_height: z.number().int().positive(),
|
||||
screen_width: z.number().int().positive(),
|
||||
screen_height: z.number().int().positive(),
|
||||
scale_factor: z.number().positive(),
|
||||
});
|
||||
|
||||
const ScreenSizeSchema = z.object({
|
||||
width: z.number().int().positive(),
|
||||
height: z.number().int().positive(),
|
||||
scale_factor: z.number().positive(),
|
||||
});
|
||||
|
||||
type ImageProcessor = {
|
||||
encode(
|
||||
input: Buffer,
|
||||
options: {
|
||||
format: "jpeg" | "png";
|
||||
quality?: number;
|
||||
resize?: { width: number; enlarge: false };
|
||||
},
|
||||
): Promise<{ data: Buffer; width: number; height: number }>;
|
||||
};
|
||||
|
||||
type CuaComputerCommandsOptions = {
|
||||
driverPath?: string;
|
||||
platform?: NodeJS.Platform;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
driver?: CuaDriver;
|
||||
imageProcessor?: ImageProcessor;
|
||||
setInterval?: typeof setInterval;
|
||||
clearInterval?: typeof clearInterval;
|
||||
};
|
||||
|
||||
class PromiseQueue {
|
||||
private tail: Promise<void> = Promise.resolve();
|
||||
|
||||
async run<T>(operation: () => Promise<T>): Promise<T> {
|
||||
const previous = this.tail;
|
||||
let release = () => {};
|
||||
this.tail = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
await previous;
|
||||
try {
|
||||
return await operation();
|
||||
} finally {
|
||||
release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function parseParams<T>(schema: z.ZodType<T>, paramsJSON: string | null | undefined): T {
|
||||
let value: unknown;
|
||||
try {
|
||||
value = JSON.parse(paramsJSON ?? "{}");
|
||||
} catch {
|
||||
throw new Error("COMPUTER_INVALID_REQUEST: params must be valid JSON");
|
||||
}
|
||||
const parsed = schema.safeParse(value);
|
||||
if (!parsed.success) {
|
||||
throw new Error(
|
||||
`COMPUTER_INVALID_REQUEST: ${parsed.error.issues[0]?.message ?? "invalid params"}`,
|
||||
);
|
||||
}
|
||||
return parsed.data;
|
||||
}
|
||||
|
||||
function assertPrimaryDisplay(screenIndex: number | undefined): void {
|
||||
if (screenIndex !== undefined && screenIndex !== 0) {
|
||||
throw new Error(
|
||||
"COMPUTER_UNSUPPORTED_DISPLAY: cua-driver controls only the primary display (screenIndex 0)",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function structuredContent(result: CuaToolResult, tool: string): Record<string, unknown> {
|
||||
if (!result.structuredContent) {
|
||||
throw new Error(`COMPUTER_DRIVER_ERROR: ${tool} returned no structuredContent`);
|
||||
}
|
||||
return result.structuredContent;
|
||||
}
|
||||
|
||||
function desktopGeometry(result: CuaToolResult): CuaDesktopGeometry {
|
||||
const parsed = DesktopStateSchema.safeParse(structuredContent(result, "get_desktop_state"));
|
||||
if (!parsed.success) {
|
||||
throw new Error("COMPUTER_DRIVER_ERROR: invalid get_desktop_state geometry");
|
||||
}
|
||||
return {
|
||||
platform: parsed.data.platform,
|
||||
display: parsed.data.display,
|
||||
screenWidth: parsed.data.screen_width,
|
||||
screenHeight: parsed.data.screen_height,
|
||||
scaleFactor: parsed.data.scale_factor,
|
||||
screenshotWidth: parsed.data.screenshot_width,
|
||||
screenshotHeight: parsed.data.screenshot_height,
|
||||
};
|
||||
}
|
||||
|
||||
function desktopPng(result: CuaToolResult): Buffer {
|
||||
const image = result.content.find(
|
||||
(entry): entry is { type: "image"; data: string; mimeType: string } =>
|
||||
entry.type === "image" && typeof entry.data === "string" && entry.mimeType === "image/png",
|
||||
);
|
||||
if (!image) {
|
||||
throw new Error("COMPUTER_DRIVER_ERROR: get_desktop_state returned no PNG image");
|
||||
}
|
||||
return Buffer.from(image.data, "base64");
|
||||
}
|
||||
|
||||
function screenSize(result: CuaToolResult): CuaScreenSize {
|
||||
const parsed = ScreenSizeSchema.safeParse(structuredContent(result, "get_screen_size"));
|
||||
if (!parsed.success) {
|
||||
throw new Error("COMPUTER_DRIVER_ERROR: invalid get_screen_size geometry");
|
||||
}
|
||||
return {
|
||||
width: parsed.data.width,
|
||||
height: parsed.data.height,
|
||||
scaleFactor: parsed.data.scale_factor,
|
||||
};
|
||||
}
|
||||
|
||||
function resolveImageCommand(command: string, env: NodeJS.ProcessEnv): string | null {
|
||||
const names =
|
||||
process.platform === "win32" && !path.extname(command)
|
||||
? [command, `${command}.exe`, `${command}.cmd`]
|
||||
: [command];
|
||||
for (const entry of (env.PATH ?? "").split(path.delimiter).filter(Boolean)) {
|
||||
for (const name of names) {
|
||||
const candidate = path.resolve(entry, name);
|
||||
try {
|
||||
fs.accessSync(candidate, fs.constants.X_OK);
|
||||
return candidate;
|
||||
} catch {
|
||||
// Continue through PATH.
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function createImageProcessor(env: NodeJS.ProcessEnv): ImageProcessor {
|
||||
return createRastermill({
|
||||
execution: "auto",
|
||||
limits: { inputPixels: MAX_IMAGE_PIXELS, outputPixels: MAX_IMAGE_PIXELS },
|
||||
temp: { rootDir: resolvePreferredOpenClawTmpDir(), prefix: "openclaw-cua-computer-" },
|
||||
commandResolver: (command) => resolveImageCommand(command, env),
|
||||
});
|
||||
}
|
||||
|
||||
function clickArgs(
|
||||
platform: NodeJS.Platform,
|
||||
frame: CuaLastFrame,
|
||||
params: ComputerActParams,
|
||||
button: "left" | "right" | "middle",
|
||||
count: 1 | 2 | 3,
|
||||
): Record<string, unknown> {
|
||||
const point = scalePoint(frame, params.x, params.y, params.action);
|
||||
const modifiers = normalizeModifiers(params.modifiers);
|
||||
if (platform === "linux" && modifiers.length > 0) {
|
||||
throw new Error(
|
||||
"COMPUTER_UNSUPPORTED_ACTION: modifier-held clicks are unsupported by cua-driver on Linux",
|
||||
);
|
||||
}
|
||||
return {
|
||||
...point,
|
||||
scope: "desktop",
|
||||
button,
|
||||
count,
|
||||
...(modifiers.length > 0 ? { modifier: modifiers } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
async function currentFrame(
|
||||
driver: CuaDriver,
|
||||
frameState: CuaFrameState,
|
||||
params: ComputerActParams,
|
||||
): Promise<CuaLastFrame> {
|
||||
const current = screenSize(await driver.callTool("get_screen_size", {}));
|
||||
if (driver.generation !== frameState.generation) {
|
||||
frameState.lastFrame = undefined;
|
||||
throw new Error("COMPUTER_STALE_FRAME: the computer driver reconnected; take a new screenshot");
|
||||
}
|
||||
const frame = verifyFrame(frameState, params.displayFrameId, current);
|
||||
verifyReferenceWidth(frameState, frame, params.refWidth);
|
||||
return frame;
|
||||
}
|
||||
|
||||
async function handleAct(
|
||||
driver: CuaDriver,
|
||||
frameState: CuaFrameState,
|
||||
params: ComputerActParams,
|
||||
platform: NodeJS.Platform,
|
||||
): Promise<string> {
|
||||
assertPrimaryDisplay(params.screenIndex);
|
||||
// `wait` never reaches the wire: core sleeps locally and the Swift wire enum
|
||||
// has no wait case, so accepting it here would fork the computer.act contract.
|
||||
if (
|
||||
params.action === "hold_key" ||
|
||||
params.action === "left_mouse_down" ||
|
||||
params.action === "left_mouse_up"
|
||||
) {
|
||||
// Upstream has no desktop keyboard-down API, and its Linux mouse hold tools
|
||||
// are window-only, so these actions cannot preserve desktop-scope semantics.
|
||||
throw new Error(`COMPUTER_UNSUPPORTED_ACTION: ${params.action}`);
|
||||
}
|
||||
|
||||
// Every action uses scope:"desktop", a global SendInput/XTest/wayland_desktop
|
||||
// injection that is inherently foreground and ignores delivery_mode (that
|
||||
// background-vs-foreground contract is window-targeted only). We deliberately
|
||||
// never send delivery_mode.
|
||||
switch (params.action) {
|
||||
case "type": {
|
||||
if (!params.text) {
|
||||
throw new Error("COMPUTER_INVALID_REQUEST: text is required for type");
|
||||
}
|
||||
await driver.callTool("type_text", { text: params.text, scope: "desktop" });
|
||||
break;
|
||||
}
|
||||
case "key": {
|
||||
// press_key applies the modifier array on every backend: X11 via XTest,
|
||||
// and native Wayland by internally promoting a modifier chord to
|
||||
// hotkey_focused. No separate hotkey call is needed for chords.
|
||||
const chord = parseKeyChord(params.keys);
|
||||
await driver.callTool("press_key", {
|
||||
key: chord.key,
|
||||
modifiers: chord.modifiers,
|
||||
scope: "desktop",
|
||||
});
|
||||
break;
|
||||
}
|
||||
case "scroll": {
|
||||
if (!params.scrollDirection) {
|
||||
throw new Error("COMPUTER_INVALID_REQUEST: scrollDirection is required for scroll");
|
||||
}
|
||||
if (normalizeModifiers(params.modifiers).length > 0) {
|
||||
throw new Error(
|
||||
"COMPUTER_UNSUPPORTED_ACTION: modifier-held scroll is unsupported by cua-driver 0.10.x",
|
||||
);
|
||||
}
|
||||
// Desktop-scope scroll requires explicit coordinates, and they must be
|
||||
// frame-authorized like clicks. We deliberately do not synthesize a point
|
||||
// from get_cursor_position: that mixes cursor and capture coordinate
|
||||
// spaces across X11/Wayland/Windows and would scroll an unverified target.
|
||||
const frame = await currentFrame(driver, frameState, params);
|
||||
const point = scalePoint(frame, params.x, params.y, params.action);
|
||||
await driver.callTool("scroll", {
|
||||
direction: params.scrollDirection,
|
||||
// Schema guarantees a positive amount; cap at the driver's max of 50.
|
||||
amount: Math.min(50, params.scrollAmount ?? 3),
|
||||
by: "line",
|
||||
...point,
|
||||
scope: "desktop",
|
||||
});
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
const frame = await currentFrame(driver, frameState, params);
|
||||
switch (params.action) {
|
||||
case "left_click":
|
||||
await driver.callTool("click", clickArgs(platform, frame, params, "left", 1));
|
||||
break;
|
||||
case "right_click":
|
||||
await driver.callTool("click", clickArgs(platform, frame, params, "right", 1));
|
||||
break;
|
||||
case "middle_click":
|
||||
await driver.callTool("click", clickArgs(platform, frame, params, "middle", 1));
|
||||
break;
|
||||
case "double_click":
|
||||
await driver.callTool("click", clickArgs(platform, frame, params, "left", 2));
|
||||
break;
|
||||
case "triple_click":
|
||||
await driver.callTool("click", clickArgs(platform, frame, params, "left", 3));
|
||||
break;
|
||||
case "mouse_move": {
|
||||
const point = scalePoint(frame, params.x, params.y, params.action);
|
||||
await driver.callTool("move_cursor", { ...point, scope: "desktop" });
|
||||
break;
|
||||
}
|
||||
case "left_click_drag": {
|
||||
const from = scalePoint(frame, params.fromX, params.fromY, "drag start");
|
||||
const to = scalePoint(frame, params.x, params.y, "drag end");
|
||||
// cua-driver 0.10 accepts `modifier` in the drag schema but its
|
||||
// desktop-scope branch never reads it (Windows impl_.rs drag desktop
|
||||
// path uses only coords/duration/steps/button), so a Shift/Ctrl-drag
|
||||
// would silently become a plain drag. Refuse instead of misfiring.
|
||||
if (normalizeModifiers(params.modifiers).length > 0) {
|
||||
throw new Error(
|
||||
"COMPUTER_UNSUPPORTED_ACTION: modifier-held drag is unsupported by cua-driver 0.10.x",
|
||||
);
|
||||
}
|
||||
await driver.callTool("drag", {
|
||||
from_x: from.x,
|
||||
from_y: from.y,
|
||||
to_x: to.x,
|
||||
to_y: to.y,
|
||||
scope: "desktop",
|
||||
// cua-driver caps drag duration_ms at 10_000; clamp so a longer
|
||||
// request runs at the max instead of being rejected at the MCP edge.
|
||||
...(params.durationMs === undefined
|
||||
? {}
|
||||
: { duration_ms: Math.min(10_000, params.durationMs) }),
|
||||
});
|
||||
break;
|
||||
}
|
||||
default:
|
||||
throw new Error("COMPUTER_UNSUPPORTED_ACTION: unknown action");
|
||||
}
|
||||
}
|
||||
}
|
||||
return JSON.stringify({ ok: true });
|
||||
}
|
||||
|
||||
export function createCuaComputerCommands(
|
||||
options: CuaComputerCommandsOptions = {},
|
||||
): OpenClawPluginNodeHostCommand[] {
|
||||
const platform = options.platform ?? process.platform;
|
||||
const env = options.env ?? process.env;
|
||||
const driver =
|
||||
options.driver ?? new CuaDriverClient({ driverPath: options.driverPath, platform, env });
|
||||
const imageProcessor = options.imageProcessor ?? createImageProcessor(env);
|
||||
const queue = new PromiseQueue();
|
||||
const frameState: CuaFrameState = { generation: driver.generation };
|
||||
const interval = options.setInterval ?? setInterval;
|
||||
const clear = options.clearInterval ?? clearInterval;
|
||||
const isSupportedPlatform = platform === "linux" || platform === "win32";
|
||||
const isAvailable = () => isSupportedPlatform && driver.isAvailable();
|
||||
|
||||
const snapshot: OpenClawPluginNodeHostCommand = {
|
||||
command: "screen.snapshot",
|
||||
cap: "screen",
|
||||
dangerous: false,
|
||||
isAvailable,
|
||||
watchAvailability: (_context, onChange) => {
|
||||
let knownAvailable = isAvailable();
|
||||
const timer = interval(() => {
|
||||
driver.resetAvailabilityCache();
|
||||
const available = isAvailable();
|
||||
if (available !== knownAvailable) {
|
||||
knownAvailable = available;
|
||||
onChange();
|
||||
}
|
||||
}, AVAILABILITY_POLL_MS);
|
||||
timer.unref?.();
|
||||
return () => {
|
||||
clear(timer);
|
||||
void driver.dispose();
|
||||
};
|
||||
},
|
||||
handle: async (paramsJSON) =>
|
||||
await queue.run(async () => {
|
||||
if (!isSupportedPlatform) {
|
||||
throw new Error("COMPUTER_DRIVER_UNAVAILABLE: cua-computer supports Windows and Linux");
|
||||
}
|
||||
const params = parseParams(SnapshotParamsSchema, paramsJSON);
|
||||
assertPrimaryDisplay(params.screenIndex);
|
||||
const format = params.format ?? "jpeg";
|
||||
const maxWidth = params.maxWidth ?? (format === "png" ? 900 : 1_600);
|
||||
const quality = Math.min(1, Math.max(0.05, params.quality ?? 0.72));
|
||||
const desktop = await driver.callTool("get_desktop_state", {});
|
||||
const geometry = desktopGeometry(desktop);
|
||||
// cua-driver desktop input consumes native get_desktop_state PNG pixels,
|
||||
// and on every supported backend the driver reports screen geometry in
|
||||
// that same physical-pixel space (Windows PMv2, Linux X11/Wayland). If a
|
||||
// capture ever diverges from screen geometry, our screenshot->native
|
||||
// scaling would mis-target input, so refuse rather than click blind.
|
||||
if (
|
||||
geometry.screenWidth !== geometry.screenshotWidth ||
|
||||
geometry.screenHeight !== geometry.screenshotHeight
|
||||
) {
|
||||
throw new Error(
|
||||
"COMPUTER_UNSUPPORTED_DISPLAY: cua-driver reported capture and screen geometry in different pixel spaces",
|
||||
);
|
||||
}
|
||||
const nativePng = desktopPng(desktop);
|
||||
let encoded = nativePng;
|
||||
let width = geometry.screenshotWidth;
|
||||
let height = geometry.screenshotHeight;
|
||||
if (format === "jpeg" || width > maxWidth) {
|
||||
const result = await imageProcessor.encode(nativePng, {
|
||||
format,
|
||||
...(format === "jpeg" ? { quality: Math.round(quality * 100) } : {}),
|
||||
...(width > maxWidth ? { resize: { width: maxWidth, enlarge: false } } : {}),
|
||||
});
|
||||
encoded = result.data;
|
||||
width = result.width;
|
||||
height = result.height;
|
||||
}
|
||||
frameState.generation = driver.generation;
|
||||
const displayFrameId = issueFrame(frameState, geometry, { width, height });
|
||||
return JSON.stringify({
|
||||
format,
|
||||
base64: encoded.toString("base64"),
|
||||
displayFrameId,
|
||||
screenIndex: 0,
|
||||
width,
|
||||
height,
|
||||
});
|
||||
}),
|
||||
};
|
||||
|
||||
const act: OpenClawPluginNodeHostCommand = {
|
||||
command: "computer.act",
|
||||
cap: "computer",
|
||||
dangerous: true,
|
||||
isAvailable,
|
||||
handle: async (paramsJSON) =>
|
||||
await queue.run(async () => {
|
||||
if (!isSupportedPlatform) {
|
||||
throw new Error("COMPUTER_DRIVER_UNAVAILABLE: cua-computer supports Windows and Linux");
|
||||
}
|
||||
return await handleAct(
|
||||
driver,
|
||||
frameState,
|
||||
parseParams(ComputerActParamsSchema, paramsJSON),
|
||||
platform,
|
||||
);
|
||||
}),
|
||||
};
|
||||
|
||||
return [snapshot, act];
|
||||
}
|
||||
@@ -0,0 +1,480 @@
|
||||
import type { ChildProcess } from "node:child_process";
|
||||
import type { Transport } from "@modelcontextprotocol/sdk/shared/transport.js";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { CuaDriverClient } from "./driver-client.js";
|
||||
|
||||
function fakeTransport(): Transport {
|
||||
return {
|
||||
start: vi.fn(async () => {}),
|
||||
send: vi.fn(async () => {}),
|
||||
close: vi.fn(async () => {}),
|
||||
};
|
||||
}
|
||||
|
||||
function fakeClient(options: {
|
||||
name?: string;
|
||||
version?: string;
|
||||
capabilityVersion?: string;
|
||||
schemaVersion?: string;
|
||||
connect?: () => Promise<void>;
|
||||
result?: unknown;
|
||||
callToolThrows?: boolean;
|
||||
}) {
|
||||
return {
|
||||
connect: vi.fn(options.connect ?? (async () => {})),
|
||||
getServerVersion: () => ({
|
||||
name: options.name ?? "cua-driver",
|
||||
version: options.version ?? "0.10.4",
|
||||
}),
|
||||
listTools: vi.fn(async () => ({
|
||||
tools: [],
|
||||
capability_version: options.capabilityVersion ?? "1",
|
||||
schema_version: options.schemaVersion ?? "1",
|
||||
})),
|
||||
callTool: vi.fn(async () => {
|
||||
if (options.callToolThrows) {
|
||||
throw new Error("transport closed");
|
||||
}
|
||||
return options.result ?? { content: [{ type: "text", text: "ok" }] };
|
||||
}),
|
||||
close: vi.fn(async () => {}),
|
||||
};
|
||||
}
|
||||
|
||||
function createClient(client: ReturnType<typeof fakeClient>) {
|
||||
return new CuaDriverClient({
|
||||
driverPath: "/opt/bin/cua-driver",
|
||||
access: () => {},
|
||||
clientFactory: () => client,
|
||||
transportFactory: fakeTransport,
|
||||
});
|
||||
}
|
||||
|
||||
describe("CuaDriverClient version gate", () => {
|
||||
it("accepts cua-driver 0.10.x with capability and schema version 1", async () => {
|
||||
const driver = createClient(fakeClient({ version: "0.10.4" }));
|
||||
await expect(driver.callTool("get_screen_size", {})).resolves.toMatchObject({
|
||||
content: [{ type: "text", text: "ok" }],
|
||||
});
|
||||
expect(driver.generation).toBe(1);
|
||||
});
|
||||
|
||||
it.each([
|
||||
[{ version: "0.11.0" }, "cua-driver@0.11.0"],
|
||||
[{ name: "other-driver" }, "other-driver@0.10.4"],
|
||||
[{ capabilityVersion: "2" }, "capability_version=2"],
|
||||
[{ schemaVersion: "2" }, "schema_version=2"],
|
||||
])("rejects unsupported initialize/list result %#", async (options, found) => {
|
||||
const driver = createClient(fakeClient(options));
|
||||
await expect(driver.callTool("get_screen_size", {})).rejects.toThrow(found);
|
||||
expect(driver.isAvailable()).toBe(false);
|
||||
});
|
||||
|
||||
it("re-probes and recovers after a corrected driver replaces an unsupported one", async () => {
|
||||
const clients = [fakeClient({ version: "0.11.0" }), fakeClient({ version: "0.10.4" })];
|
||||
let clock = 1_000;
|
||||
const driver = new CuaDriverClient({
|
||||
driverPath: "/opt/bin/cua-driver",
|
||||
access: () => {},
|
||||
clientFactory: () => clients.shift() ?? fakeClient({ version: "0.10.4" }),
|
||||
transportFactory: fakeTransport,
|
||||
now: () => clock,
|
||||
});
|
||||
|
||||
await expect(driver.callTool("get_screen_size", {})).rejects.toThrow("cua-driver@0.11.0");
|
||||
// Within the re-probe window the verdict is cached: unavailable, no reconnect.
|
||||
clock += 10_000;
|
||||
expect(driver.isAvailable()).toBe(false);
|
||||
// After the window, the corrected driver is re-probed and accepted.
|
||||
clock += 25_000;
|
||||
expect(driver.isAvailable()).toBe(true);
|
||||
await expect(driver.callTool("get_screen_size", {})).resolves.toMatchObject({
|
||||
content: [{ type: "text", text: "ok" }],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("CuaDriverClient process contract", () => {
|
||||
it("forwards only allowlisted env, opt-outs, and the Wayland setting, dropping secrets", async () => {
|
||||
let transportParams: { env?: Record<string, string>; stderr?: unknown } | undefined;
|
||||
const driver = new CuaDriverClient({
|
||||
driverPath: "/opt/bin/cua-driver",
|
||||
env: {
|
||||
PATH: "/opt/bin",
|
||||
DISPLAY: ":0",
|
||||
XDG_RUNTIME_DIR: "/run/user/1000",
|
||||
LC_ALL: "en_US.UTF-8",
|
||||
CUA_DRIVER_RS_ENABLE_WAYLAND: "1",
|
||||
OPENAI_API_KEY: "sk-should-not-leak",
|
||||
ANTHROPIC_API_KEY: "should-not-leak",
|
||||
SLACK_BOT_TOKEN: "xoxb-should-not-leak",
|
||||
CUA_API_KEY: "cua-cloud-should-not-leak",
|
||||
},
|
||||
access: () => {},
|
||||
clientFactory: () => fakeClient({}),
|
||||
transportFactory: (params) => {
|
||||
transportParams = params;
|
||||
return fakeTransport();
|
||||
},
|
||||
});
|
||||
|
||||
await driver.callTool("get_screen_size", {});
|
||||
|
||||
expect(transportParams?.env).toMatchObject({
|
||||
PATH: "/opt/bin",
|
||||
DISPLAY: ":0",
|
||||
XDG_RUNTIME_DIR: "/run/user/1000",
|
||||
LC_ALL: "en_US.UTF-8",
|
||||
CUA_DRIVER_RS_TELEMETRY_ENABLED: "false",
|
||||
CUA_DRIVER_RS_UPDATE_CHECK: "false",
|
||||
CUA_DRIVER_RS_ENABLE_WAYLAND: "1",
|
||||
});
|
||||
expect(transportParams?.env).not.toHaveProperty("OPENAI_API_KEY");
|
||||
expect(transportParams?.env).not.toHaveProperty("ANTHROPIC_API_KEY");
|
||||
expect(transportParams?.env).not.toHaveProperty("SLACK_BOT_TOKEN");
|
||||
// The CUA_ namespace holds cloud credentials, so it is not prefix-allowed.
|
||||
expect(transportParams?.env).not.toHaveProperty("CUA_API_KEY");
|
||||
expect(transportParams?.stderr).toBe("ignore");
|
||||
});
|
||||
|
||||
it("starts serve and connects on the first readiness poll", async () => {
|
||||
const first = fakeClient({
|
||||
connect: async () => {
|
||||
throw new Error("daemon absent");
|
||||
},
|
||||
});
|
||||
const second = fakeClient({});
|
||||
const clients = [first, second];
|
||||
const kill = vi.fn();
|
||||
const child = {
|
||||
exitCode: null,
|
||||
once: vi.fn(),
|
||||
unref: vi.fn(),
|
||||
kill,
|
||||
} as unknown as ChildProcess;
|
||||
const spawnProcess = vi.fn(() => child);
|
||||
const driver = new CuaDriverClient({
|
||||
driverPath: "/opt/bin/cua-driver",
|
||||
env: { PATH: "/opt/bin" },
|
||||
access: () => {},
|
||||
clientFactory: () => clients.shift() ?? second,
|
||||
transportFactory: fakeTransport,
|
||||
spawn: spawnProcess as unknown as typeof import("node:child_process").spawn,
|
||||
sleep: async () => {},
|
||||
});
|
||||
|
||||
await driver.callTool("get_screen_size", {});
|
||||
expect(spawnProcess).toHaveBeenCalledWith(
|
||||
"/opt/bin/cua-driver",
|
||||
["serve"],
|
||||
expect.objectContaining({
|
||||
detached: true,
|
||||
env: expect.objectContaining({
|
||||
CUA_DRIVER_RS_TELEMETRY_ENABLED: "false",
|
||||
CUA_DRIVER_RS_UPDATE_CHECK: "false",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
await driver.dispose();
|
||||
// The shared machine daemon must outlive our client; dispose closes the mcp
|
||||
// session but never kills serve, so other cua-driver clients stay connected.
|
||||
expect(kill).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps polling through a slow daemon start and records the backoff delays", async () => {
|
||||
const failing = () =>
|
||||
fakeClient({
|
||||
connect: async () => {
|
||||
throw new Error("daemon still starting");
|
||||
},
|
||||
});
|
||||
const clients = [failing(), failing(), failing(), fakeClient({})];
|
||||
const delays: number[] = [];
|
||||
const child = {
|
||||
exitCode: null,
|
||||
once: vi.fn(),
|
||||
unref: vi.fn(),
|
||||
kill: vi.fn(),
|
||||
} as unknown as ChildProcess;
|
||||
const driver = new CuaDriverClient({
|
||||
driverPath: "/opt/bin/cua-driver",
|
||||
env: { PATH: "/opt/bin" },
|
||||
access: () => {},
|
||||
clientFactory: () => clients.shift() ?? fakeClient({}),
|
||||
transportFactory: fakeTransport,
|
||||
spawn: vi.fn(() => child) as unknown as typeof import("node:child_process").spawn,
|
||||
sleep: async (durationMs) => {
|
||||
delays.push(durationMs);
|
||||
},
|
||||
});
|
||||
|
||||
await driver.callTool("get_screen_size", {});
|
||||
expect(delays).toEqual([250, 500, 1_000]);
|
||||
});
|
||||
|
||||
it("keeps polling the full budget even after the spawned child exits", async () => {
|
||||
const delays: number[] = [];
|
||||
const child = {
|
||||
exitCode: null,
|
||||
once: vi.fn(),
|
||||
unref: vi.fn(),
|
||||
kill: vi.fn(),
|
||||
} as unknown as ChildProcess;
|
||||
const driver = new CuaDriverClient({
|
||||
driverPath: "/opt/bin/cua-driver",
|
||||
env: { PATH: "/opt/bin" },
|
||||
access: () => {},
|
||||
clientFactory: () =>
|
||||
fakeClient({
|
||||
connect: async () => {
|
||||
throw new Error("daemon absent");
|
||||
},
|
||||
}),
|
||||
transportFactory: fakeTransport,
|
||||
spawn: vi.fn(() => child) as unknown as typeof import("node:child_process").spawn,
|
||||
sleep: async (durationMs) => {
|
||||
delays.push(durationMs);
|
||||
(child as { exitCode: number | null }).exitCode = 3;
|
||||
},
|
||||
});
|
||||
|
||||
await expect(driver.callTool("get_screen_size", {})).rejects.toThrow(
|
||||
/COMPUTER_DRIVER_UNAVAILABLE: cua-driver daemon did not become ready in time/,
|
||||
);
|
||||
// Child exit must not short-circuit the schedule.
|
||||
expect(delays).toEqual([250, 500, 1_000, 2_000, 3_000, 3_000]);
|
||||
});
|
||||
|
||||
it("respawns the daemon when the remembered child was signal-terminated", async () => {
|
||||
// Two connect cycles: cycle 1 spawns child0 and caches a session whose
|
||||
// callTool then breaks; cycle 2 must spawn again because child0 was killed
|
||||
// by signal (exitCode null, signalCode set) rather than a clean exit.
|
||||
const clients = [
|
||||
fakeClient({
|
||||
connect: async () => {
|
||||
throw new Error("daemon absent");
|
||||
},
|
||||
}),
|
||||
fakeClient({ callToolThrows: true }),
|
||||
fakeClient({
|
||||
connect: async () => {
|
||||
throw new Error("daemon absent");
|
||||
},
|
||||
}),
|
||||
fakeClient({}),
|
||||
];
|
||||
const spawned: Array<{ signalCode: string | null; exitCode: number | null }> = [];
|
||||
const spawnProcess = vi.fn(() => {
|
||||
const child = {
|
||||
exitCode: null,
|
||||
signalCode: null,
|
||||
once: vi.fn(),
|
||||
unref: vi.fn(),
|
||||
kill: vi.fn(),
|
||||
};
|
||||
spawned.push(child);
|
||||
return child as unknown as ChildProcess;
|
||||
});
|
||||
const driver = new CuaDriverClient({
|
||||
driverPath: "/opt/bin/cua-driver",
|
||||
env: { PATH: "/opt/bin" },
|
||||
access: () => {},
|
||||
clientFactory: () => clients.shift() ?? fakeClient({}),
|
||||
transportFactory: fakeTransport,
|
||||
spawn: spawnProcess as unknown as typeof import("node:child_process").spawn,
|
||||
sleep: async () => {},
|
||||
});
|
||||
|
||||
await expect(driver.callTool("get_screen_size", {})).rejects.toThrow("transport closed");
|
||||
const firstChild = spawned[0];
|
||||
expect(firstChild).toBeDefined();
|
||||
firstChild!.signalCode = "SIGKILL";
|
||||
await expect(driver.callTool("get_desktop_state", {})).resolves.toBeDefined();
|
||||
expect(spawnProcess).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("respawns after the spawned child emits an async spawn error", async () => {
|
||||
// `error` can fire without `exit`, leaving exitCode/signalCode null; the
|
||||
// handler must still forget the child so the next connect respawns.
|
||||
const clients = [
|
||||
fakeClient({
|
||||
connect: async () => {
|
||||
throw new Error("daemon absent");
|
||||
},
|
||||
}),
|
||||
fakeClient({ callToolThrows: true }),
|
||||
fakeClient({
|
||||
connect: async () => {
|
||||
throw new Error("daemon absent");
|
||||
},
|
||||
}),
|
||||
fakeClient({}),
|
||||
];
|
||||
const errorHandlers: Array<() => void> = [];
|
||||
const spawnProcess = vi.fn(() => {
|
||||
const child = {
|
||||
exitCode: null,
|
||||
signalCode: null,
|
||||
unref: vi.fn(),
|
||||
kill: vi.fn(),
|
||||
once: (event: string, cb: () => void) => {
|
||||
if (event === "error") {
|
||||
errorHandlers.push(cb);
|
||||
}
|
||||
},
|
||||
};
|
||||
return child as unknown as ChildProcess;
|
||||
});
|
||||
const driver = new CuaDriverClient({
|
||||
driverPath: "/opt/bin/cua-driver",
|
||||
env: { PATH: "/opt/bin" },
|
||||
access: () => {},
|
||||
clientFactory: () => clients.shift() ?? fakeClient({}),
|
||||
transportFactory: fakeTransport,
|
||||
spawn: spawnProcess as unknown as typeof import("node:child_process").spawn,
|
||||
sleep: async () => {},
|
||||
});
|
||||
|
||||
await expect(driver.callTool("get_screen_size", {})).rejects.toThrow("transport closed");
|
||||
errorHandlers[0]?.();
|
||||
await expect(driver.callTool("get_desktop_state", {})).resolves.toBeDefined();
|
||||
expect(spawnProcess).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("connects to a shared daemon even when our serve child lost the startup race", async () => {
|
||||
const clients = [
|
||||
fakeClient({
|
||||
connect: async () => {
|
||||
throw new Error("daemon absent");
|
||||
},
|
||||
}),
|
||||
fakeClient({}),
|
||||
];
|
||||
const child = {
|
||||
exitCode: null,
|
||||
once: vi.fn(),
|
||||
unref: vi.fn(),
|
||||
kill: vi.fn(),
|
||||
} as unknown as ChildProcess;
|
||||
const driver = new CuaDriverClient({
|
||||
driverPath: "/opt/bin/cua-driver",
|
||||
env: { PATH: "/opt/bin" },
|
||||
access: () => {},
|
||||
clientFactory: () => clients.shift() ?? fakeClient({}),
|
||||
transportFactory: fakeTransport,
|
||||
spawn: vi.fn(() => child) as unknown as typeof import("node:child_process").spawn,
|
||||
// Our serve child exited (another client won the race), but the shared
|
||||
// daemon it collided with is now serving; the retry must still connect.
|
||||
sleep: async () => {
|
||||
(child as { exitCode: number | null }).exitCode = 1;
|
||||
},
|
||||
});
|
||||
|
||||
await expect(driver.callTool("get_screen_size", {})).resolves.toBeDefined();
|
||||
});
|
||||
|
||||
it("gives up with a readiness timeout after exhausting the backoff schedule", async () => {
|
||||
const delays: number[] = [];
|
||||
const child = {
|
||||
exitCode: null,
|
||||
once: vi.fn(),
|
||||
unref: vi.fn(),
|
||||
kill: vi.fn(),
|
||||
} as unknown as ChildProcess;
|
||||
const driver = new CuaDriverClient({
|
||||
driverPath: "/opt/bin/cua-driver",
|
||||
env: { PATH: "/opt/bin" },
|
||||
access: () => {},
|
||||
clientFactory: () =>
|
||||
fakeClient({
|
||||
connect: async () => {
|
||||
throw new Error("daemon never ready");
|
||||
},
|
||||
}),
|
||||
transportFactory: fakeTransport,
|
||||
spawn: vi.fn(() => child) as unknown as typeof import("node:child_process").spawn,
|
||||
sleep: async (durationMs) => {
|
||||
delays.push(durationMs);
|
||||
},
|
||||
});
|
||||
|
||||
await expect(driver.callTool("get_screen_size", {})).rejects.toThrow(
|
||||
/COMPUTER_DRIVER_UNAVAILABLE: cua-driver daemon did not become ready in time/,
|
||||
);
|
||||
expect(delays).toEqual([250, 500, 1_000, 2_000, 3_000, 3_000]);
|
||||
});
|
||||
|
||||
it("closes a connection that completes after disposal starts", async () => {
|
||||
let finishConnect = () => {};
|
||||
const connecting = new Promise<void>((resolve) => {
|
||||
finishConnect = resolve;
|
||||
});
|
||||
const client = fakeClient({ connect: async () => await connecting });
|
||||
const driver = createClient(client);
|
||||
|
||||
const call = driver.callTool("get_screen_size", {});
|
||||
const disposal = driver.dispose();
|
||||
finishConnect();
|
||||
|
||||
await expect(call).rejects.toThrow("cua-driver client is disposed");
|
||||
await disposal;
|
||||
expect(client.callTool).not.toHaveBeenCalled();
|
||||
expect(client.close).toHaveBeenCalled();
|
||||
expect(driver.isAvailable()).toBe(false);
|
||||
});
|
||||
|
||||
it("caches binary resolution for one second", () => {
|
||||
let now = 0;
|
||||
const access = vi.fn(() => {});
|
||||
const driver = new CuaDriverClient({
|
||||
env: { PATH: "/one:/two" },
|
||||
now: () => now,
|
||||
access,
|
||||
});
|
||||
expect(driver.isAvailable()).toBe(true);
|
||||
expect(driver.isAvailable()).toBe(true);
|
||||
expect(access).toHaveBeenCalledTimes(1);
|
||||
now = 1_001;
|
||||
expect(driver.isAvailable()).toBe(true);
|
||||
expect(access).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("reports unavailable when the binary cannot be resolved", () => {
|
||||
const driver = new CuaDriverClient({
|
||||
env: { PATH: "/missing" },
|
||||
access: () => {
|
||||
throw new Error("ENOENT");
|
||||
},
|
||||
});
|
||||
expect(driver.isAvailable()).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("CuaDriverClient refusal mapping", () => {
|
||||
it("uses a structured refusal code and the first text block", async () => {
|
||||
const driver = createClient(
|
||||
fakeClient({
|
||||
result: {
|
||||
isError: true,
|
||||
content: [
|
||||
{ type: "text", text: "desktop unavailable" },
|
||||
{ type: "text", text: "ignored" },
|
||||
],
|
||||
structuredContent: { code: "background_unavailable" },
|
||||
},
|
||||
}),
|
||||
);
|
||||
await expect(driver.callTool("click", {})).rejects.toThrow(
|
||||
"COMPUTER_REFUSED_background_unavailable: desktop unavailable",
|
||||
);
|
||||
});
|
||||
|
||||
it("falls back to the generic driver error prefix", async () => {
|
||||
const driver = createClient(
|
||||
fakeClient({
|
||||
result: { isError: true, content: [{ type: "text", text: "bad input" }] },
|
||||
}),
|
||||
);
|
||||
await expect(driver.callTool("click", {})).rejects.toThrow("COMPUTER_DRIVER_ERROR: bad input");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,514 @@
|
||||
import { spawn, type ChildProcess } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
||||
import {
|
||||
StdioClientTransport,
|
||||
type StdioServerParameters,
|
||||
} from "@modelcontextprotocol/sdk/client/stdio.js";
|
||||
import type { Transport } from "@modelcontextprotocol/sdk/shared/transport.js";
|
||||
|
||||
// cua-driver is prerelease upstream; pin the exact minor contract until it stabilizes.
|
||||
const SUPPORTED_DRIVER_VERSION_PREFIX = "0.10.";
|
||||
const BINARY_CACHE_MS = 1_000;
|
||||
// Cumulative ~9.75s of daemon readiness polling after spawning `serve`.
|
||||
const DAEMON_READY_BACKOFF_MS = [250, 500, 1_000, 2_000, 3_000, 3_000] as const;
|
||||
// How long an unsupported-version verdict suppresses re-probes. Bounded so that
|
||||
// installing the right driver or restarting an incompatible daemon recovers
|
||||
// without a node restart, while a persistently-wrong driver is not re-probed on
|
||||
// every call.
|
||||
const UNSUPPORTED_REPROBE_MS = 30_000;
|
||||
|
||||
type CuaToolContent =
|
||||
| { type: "text"; text: string }
|
||||
| { type: "image"; data: string; mimeType: string }
|
||||
| Record<string, unknown>;
|
||||
|
||||
export type CuaToolResult = {
|
||||
content: CuaToolContent[];
|
||||
isError?: boolean;
|
||||
structuredContent?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
export interface CuaDriver {
|
||||
readonly generation: number;
|
||||
isAvailable(): boolean;
|
||||
resetAvailabilityCache(): void;
|
||||
callTool(name: string, args: Record<string, unknown>): Promise<CuaToolResult>;
|
||||
dispose(): Promise<void>;
|
||||
}
|
||||
|
||||
type McpClientLike = {
|
||||
connect(transport: Transport): Promise<void>;
|
||||
getServerVersion(): { name: string; version: string } | undefined;
|
||||
listTools(): Promise<unknown>;
|
||||
callTool(params: { name: string; arguments?: Record<string, unknown> }): Promise<unknown>;
|
||||
close(): Promise<void>;
|
||||
};
|
||||
|
||||
type CuaDriverClientOptions = {
|
||||
driverPath?: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
platform?: NodeJS.Platform;
|
||||
now?: () => number;
|
||||
access?: (filePath: string, mode: number) => void;
|
||||
spawn?: typeof spawn;
|
||||
transportFactory?: (params: StdioServerParameters) => Transport;
|
||||
clientFactory?: () => McpClientLike;
|
||||
sleep?: (durationMs: number) => Promise<void>;
|
||||
};
|
||||
|
||||
type DriverSession = {
|
||||
client: McpClientLike;
|
||||
transport: Transport;
|
||||
};
|
||||
|
||||
class ComputerDriverUnsupportedError extends Error {
|
||||
readonly code = "COMPUTER_DRIVER_UNSUPPORTED";
|
||||
|
||||
constructor(found: string, pinned: string) {
|
||||
super(`COMPUTER_DRIVER_UNSUPPORTED: found ${found}; required ${pinned}`);
|
||||
this.name = "ComputerDriverUnsupportedError";
|
||||
}
|
||||
}
|
||||
|
||||
// cua-driver is a separately installed process that outlives this client, so it
|
||||
// must never inherit OpenClaw secrets (provider tokens, channel credentials).
|
||||
// Forward a deny-by-default allowlist of only the OS/session variables the
|
||||
// driver needs plus its own CUA_/XDG_/LC_ namespaces.
|
||||
const DRIVER_ENV_ALLOWLIST = new Set(
|
||||
[
|
||||
"PATH",
|
||||
"HOME",
|
||||
"USER",
|
||||
"LOGNAME",
|
||||
"USERNAME",
|
||||
"USERDOMAIN",
|
||||
"LANG",
|
||||
"LANGUAGE",
|
||||
"TERM",
|
||||
"TZ",
|
||||
"SHELL",
|
||||
"TMPDIR",
|
||||
"TEMP",
|
||||
"TMP",
|
||||
// Linux X11/Wayland session
|
||||
"DISPLAY",
|
||||
"WAYLAND_DISPLAY",
|
||||
"XAUTHORITY",
|
||||
"DBUS_SESSION_BUS_ADDRESS",
|
||||
// Windows system paths the driver's runtime relies on
|
||||
"USERPROFILE",
|
||||
"HOMEDRIVE",
|
||||
"HOMEPATH",
|
||||
"APPDATA",
|
||||
"LOCALAPPDATA",
|
||||
"PROGRAMDATA",
|
||||
"PROGRAMFILES",
|
||||
"PROGRAMFILES(X86)",
|
||||
"COMMONPROGRAMFILES",
|
||||
"COMMONPROGRAMFILES(X86)",
|
||||
"SYSTEMROOT",
|
||||
"SYSTEMDRIVE",
|
||||
"WINDIR",
|
||||
"COMSPEC",
|
||||
"PATHEXT",
|
||||
"PROCESSOR_ARCHITECTURE",
|
||||
"NUMBER_OF_PROCESSORS",
|
||||
"COMPUTERNAME",
|
||||
"SESSIONNAME",
|
||||
// cua-driver local config — an explicit list, not a CUA_ prefix, because the
|
||||
// CUA_ namespace also holds cloud credentials like CUA_API_KEY that this
|
||||
// local desktop driver never needs.
|
||||
"CUA_DRIVER_RS_ENABLE_WAYLAND",
|
||||
"CUA_DRIVER_RS_SESSION_IDLE_TTL_SECS",
|
||||
"CUA_DRIVER_POLICY_FILE",
|
||||
"CUA_DRIVER_MANAGED_POLICY_FILE",
|
||||
"CUA_DRIVER_SESSION_POLICY_FILE",
|
||||
].map((name) => name.toUpperCase()),
|
||||
);
|
||||
|
||||
// Locale and freedesktop session-dir namespaces only. Both are credential-free
|
||||
// by spec; the CUA_ namespace is deliberately excluded (see the allowlist).
|
||||
const DRIVER_ENV_ALLOW_PREFIXES = ["XDG_", "LC_"];
|
||||
|
||||
function buildDriverEnvironment(env: NodeJS.ProcessEnv): Record<string, string> {
|
||||
const result: Record<string, string> = {};
|
||||
for (const [key, value] of Object.entries(env)) {
|
||||
if (typeof value !== "string") {
|
||||
continue;
|
||||
}
|
||||
const upper = key.toUpperCase();
|
||||
const allowed =
|
||||
DRIVER_ENV_ALLOWLIST.has(upper) ||
|
||||
DRIVER_ENV_ALLOW_PREFIXES.some((prefix) => upper.startsWith(prefix));
|
||||
if (allowed) {
|
||||
result[key] = value;
|
||||
}
|
||||
}
|
||||
// Force OpenClaw-managed opt-outs even over an inherited CUA_* value.
|
||||
result.CUA_DRIVER_RS_TELEMETRY_ENABLED = "false";
|
||||
result.CUA_DRIVER_RS_UPDATE_CHECK = "false";
|
||||
return result;
|
||||
}
|
||||
|
||||
function firstTextBlock(content: CuaToolContent[]): string {
|
||||
const block = content.find(
|
||||
(entry): entry is { type: "text"; text: string } =>
|
||||
entry.type === "text" && typeof entry.text === "string",
|
||||
);
|
||||
return block?.text ?? "cua-driver tool failed";
|
||||
}
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> {
|
||||
return value && typeof value === "object" && !Array.isArray(value)
|
||||
? (value as Record<string, unknown>)
|
||||
: {};
|
||||
}
|
||||
|
||||
function asToolResult(value: unknown): CuaToolResult {
|
||||
const record = asRecord(value);
|
||||
return {
|
||||
content: Array.isArray(record.content) ? (record.content as CuaToolContent[]) : [],
|
||||
isError: record.isError === true,
|
||||
structuredContent:
|
||||
record.structuredContent && typeof record.structuredContent === "object"
|
||||
? (record.structuredContent as Record<string, unknown>)
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
export class CuaDriverClient implements CuaDriver {
|
||||
private readonly driverPath?: string;
|
||||
private readonly env: NodeJS.ProcessEnv;
|
||||
private readonly platform: NodeJS.Platform;
|
||||
private readonly now: () => number;
|
||||
private readonly access: (filePath: string, mode: number) => void;
|
||||
private readonly spawnProcess: typeof spawn;
|
||||
private readonly transportFactory: (params: StdioServerParameters) => Transport;
|
||||
private readonly clientFactory: () => McpClientLike;
|
||||
private readonly sleep: (durationMs: number) => Promise<void>;
|
||||
private binaryCache: { checkedAt: number; path: string | null } = {
|
||||
checkedAt: Number.NEGATIVE_INFINITY,
|
||||
path: null,
|
||||
};
|
||||
private session?: DriverSession;
|
||||
private connectPromise?: Promise<DriverSession>;
|
||||
private serveProcess?: ChildProcess;
|
||||
private unsupportedError?: ComputerDriverUnsupportedError;
|
||||
private unsupportedAt = 0;
|
||||
private generationValue = 0;
|
||||
private disposed = false;
|
||||
|
||||
constructor(options: CuaDriverClientOptions = {}) {
|
||||
this.driverPath = options.driverPath;
|
||||
this.env = options.env ?? process.env;
|
||||
this.platform = options.platform ?? process.platform;
|
||||
this.now = options.now ?? Date.now;
|
||||
this.access = options.access ?? fs.accessSync;
|
||||
this.spawnProcess = options.spawn ?? spawn;
|
||||
this.transportFactory =
|
||||
options.transportFactory ?? ((params) => new StdioClientTransport(params));
|
||||
this.clientFactory =
|
||||
options.clientFactory ??
|
||||
(() => new Client({ name: "openclaw-cua-computer", version: "0.0.0" }));
|
||||
this.sleep =
|
||||
options.sleep ??
|
||||
(async (durationMs) => {
|
||||
await new Promise<void>((resolve) => {
|
||||
const timer = setTimeout(resolve, durationMs);
|
||||
timer.unref?.();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
get generation(): number {
|
||||
return this.generationValue;
|
||||
}
|
||||
|
||||
private executableNames(name: string): string[] {
|
||||
if (this.platform !== "win32" || path.extname(name)) {
|
||||
return [name];
|
||||
}
|
||||
const extensions = (this.env.PATHEXT ?? ".EXE;.CMD;.BAT;.COM").split(";").filter(Boolean);
|
||||
return [name, ...extensions.map((extension) => `${name}${extension.toLowerCase()}`)];
|
||||
}
|
||||
|
||||
private canExecute(candidate: string): boolean {
|
||||
try {
|
||||
this.access(candidate, fs.constants.X_OK);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private resolveBinaryUncached(): string | null {
|
||||
const requested = this.driverPath ?? "cua-driver";
|
||||
if (path.isAbsolute(requested)) {
|
||||
return this.canExecute(requested) ? requested : null;
|
||||
}
|
||||
const pathEntries = (this.env.PATH ?? "").split(path.delimiter).filter(Boolean);
|
||||
for (const entry of pathEntries) {
|
||||
for (const name of this.executableNames(requested)) {
|
||||
const candidate = path.resolve(entry, name);
|
||||
if (this.canExecute(candidate)) {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private resolveBinary(): string | null {
|
||||
const now = this.now();
|
||||
if (now - this.binaryCache.checkedAt < BINARY_CACHE_MS) {
|
||||
return this.binaryCache.path;
|
||||
}
|
||||
const resolved = this.resolveBinaryUncached();
|
||||
this.binaryCache = { checkedAt: now, path: resolved };
|
||||
return resolved;
|
||||
}
|
||||
|
||||
/** The cached version-incompatibility error while its re-probe window holds. */
|
||||
private activeUnsupportedError(): ComputerDriverUnsupportedError | undefined {
|
||||
if (
|
||||
this.unsupportedError !== undefined &&
|
||||
this.now() - this.unsupportedAt < UNSUPPORTED_REPROBE_MS
|
||||
) {
|
||||
return this.unsupportedError;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
isAvailable(): boolean {
|
||||
return (
|
||||
!this.disposed && this.activeUnsupportedError() === undefined && this.resolveBinary() !== null
|
||||
);
|
||||
}
|
||||
|
||||
resetAvailabilityCache(): void {
|
||||
this.binaryCache.checkedAt = Number.NEGATIVE_INFINITY;
|
||||
}
|
||||
|
||||
private driverEnv(): Record<string, string> {
|
||||
return buildDriverEnvironment(this.env);
|
||||
}
|
||||
|
||||
private async closeSession(session: DriverSession | undefined): Promise<void> {
|
||||
if (!session) {
|
||||
return;
|
||||
}
|
||||
await session.client.close().catch(() => {});
|
||||
await session.transport.close().catch(() => {});
|
||||
}
|
||||
|
||||
private async connectOnce(binary: string): Promise<DriverSession> {
|
||||
const transport = this.transportFactory({
|
||||
command: binary,
|
||||
args: ["mcp"],
|
||||
env: this.driverEnv(),
|
||||
stderr: "ignore",
|
||||
});
|
||||
const client = this.clientFactory();
|
||||
const session = { client, transport };
|
||||
try {
|
||||
await client.connect(transport);
|
||||
const serverInfo = client.getServerVersion();
|
||||
const foundServer = serverInfo
|
||||
? `${serverInfo.name}@${serverInfo.version}`
|
||||
: "missing serverInfo";
|
||||
if (
|
||||
serverInfo?.name !== "cua-driver" ||
|
||||
!serverInfo.version.startsWith(SUPPORTED_DRIVER_VERSION_PREFIX)
|
||||
) {
|
||||
throw new ComputerDriverUnsupportedError(
|
||||
foundServer,
|
||||
`cua-driver@${SUPPORTED_DRIVER_VERSION_PREFIX}x`,
|
||||
);
|
||||
}
|
||||
const listed = asRecord(await client.listTools());
|
||||
const capabilityVersion = listed.capability_version;
|
||||
const schemaVersion = listed.schema_version;
|
||||
if (capabilityVersion !== "1" || schemaVersion !== "1") {
|
||||
throw new ComputerDriverUnsupportedError(
|
||||
`cua-driver@${serverInfo.version} capability_version=${String(capabilityVersion)} schema_version=${String(schemaVersion)}`,
|
||||
`cua-driver@${SUPPORTED_DRIVER_VERSION_PREFIX}x capability_version=1 schema_version=1`,
|
||||
);
|
||||
}
|
||||
this.generationValue += 1;
|
||||
return session;
|
||||
} catch (error) {
|
||||
await this.closeSession(session);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private spawnDaemon(binary: string): void {
|
||||
// A signal-terminated child leaves exitCode null but sets signalCode, so
|
||||
// both must be null to treat the remembered daemon as still running;
|
||||
// otherwise a SIGKILL/OOM'd daemon would block every future respawn.
|
||||
if (
|
||||
this.serveProcess &&
|
||||
this.serveProcess.exitCode === null &&
|
||||
this.serveProcess.signalCode == null
|
||||
) {
|
||||
return;
|
||||
}
|
||||
const child = this.spawnProcess(binary, ["serve"], {
|
||||
detached: true,
|
||||
env: this.driverEnv(),
|
||||
stdio: "ignore",
|
||||
windowsHide: true,
|
||||
});
|
||||
const forget = () => {
|
||||
if (this.serveProcess === child) {
|
||||
this.serveProcess = undefined;
|
||||
}
|
||||
};
|
||||
// A binary can disappear between the availability check and spawn. `error`
|
||||
// can fire without `exit`, leaving exitCode/signalCode both null, so forget
|
||||
// the child here too or the guard above would treat the failed spawn as a
|
||||
// live daemon forever. The MCP retry owns the actionable failure.
|
||||
child.once("error", forget);
|
||||
// Forget the child once it dies (either code or signal) so the next connect
|
||||
// spawns a fresh daemon instead of trusting a stale handle.
|
||||
child.once("exit", forget);
|
||||
child.unref();
|
||||
this.serveProcess = child;
|
||||
}
|
||||
|
||||
private async connect(): Promise<DriverSession> {
|
||||
if (this.disposed) {
|
||||
throw new Error("COMPUTER_DRIVER_UNAVAILABLE: cua-driver client is disposed");
|
||||
}
|
||||
if (this.session) {
|
||||
return this.session;
|
||||
}
|
||||
const activeUnsupported = this.activeUnsupportedError();
|
||||
if (activeUnsupported) {
|
||||
throw activeUnsupported;
|
||||
}
|
||||
// Verdict expired: allow one fresh compatibility probe so a corrected driver
|
||||
// or restarted daemon recovers without a node restart.
|
||||
this.unsupportedError = undefined;
|
||||
if (this.connectPromise) {
|
||||
return await this.connectPromise;
|
||||
}
|
||||
const binary = this.resolveBinary();
|
||||
if (!binary) {
|
||||
throw new Error("COMPUTER_DRIVER_UNAVAILABLE: cua-driver executable not found");
|
||||
}
|
||||
const pending = (async () => {
|
||||
try {
|
||||
return await this.connectOnce(binary);
|
||||
} catch (error) {
|
||||
if (error instanceof ComputerDriverUnsupportedError) {
|
||||
this.unsupportedError = error;
|
||||
this.unsupportedAt = this.now();
|
||||
throw error;
|
||||
}
|
||||
if (this.disposed) {
|
||||
throw new Error("COMPUTER_DRIVER_UNAVAILABLE: cua-driver client is disposed", {
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
this.spawnDaemon(binary);
|
||||
// A cold `serve` start (Xvfb, portals, UIA warmup) can take seconds;
|
||||
// upstream's own mcp launcher waits up to 10s for the macOS daemon.
|
||||
// Poll with backoff instead of racing one fixed delay.
|
||||
const lastIndex = DAEMON_READY_BACKOFF_MS.length - 1;
|
||||
for (const [index, delayMs] of DAEMON_READY_BACKOFF_MS.entries()) {
|
||||
await this.sleep(delayMs);
|
||||
if (this.disposed) {
|
||||
throw new Error("COMPUTER_DRIVER_UNAVAILABLE: cua-driver client is disposed", {
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
try {
|
||||
return await this.connectOnce(binary);
|
||||
} catch (retryError) {
|
||||
if (retryError instanceof ComputerDriverUnsupportedError) {
|
||||
this.unsupportedError = retryError;
|
||||
this.unsupportedAt = this.now();
|
||||
throw retryError;
|
||||
}
|
||||
// Give up only after the budget is exhausted, reporting the final
|
||||
// retry failure (the most relevant cause) rather than the first.
|
||||
// A child exit mid-budget is not terminal: cua-driver allows one
|
||||
// daemon per endpoint, so ours may have collided with a shared one
|
||||
// that needs more time to answer.
|
||||
if (index === lastIndex) {
|
||||
throw new Error(
|
||||
"COMPUTER_DRIVER_UNAVAILABLE: cua-driver daemon did not become ready in time",
|
||||
{ cause: retryError },
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Unreachable: the final iteration always returns or throws. Present for
|
||||
// control-flow completeness only.
|
||||
throw new Error("COMPUTER_DRIVER_UNAVAILABLE: cua-driver daemon did not become ready", {
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
})();
|
||||
this.connectPromise = pending;
|
||||
try {
|
||||
const session = await pending;
|
||||
if (this.disposed) {
|
||||
await this.closeSession(session);
|
||||
throw new Error("COMPUTER_DRIVER_UNAVAILABLE: cua-driver client is disposed");
|
||||
}
|
||||
this.session = session;
|
||||
return this.session;
|
||||
} finally {
|
||||
this.connectPromise = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
async callTool(name: string, args: Record<string, unknown>): Promise<CuaToolResult> {
|
||||
const session = await this.connect();
|
||||
if (this.disposed) {
|
||||
throw new Error("COMPUTER_DRIVER_UNAVAILABLE: cua-driver client is disposed");
|
||||
}
|
||||
let result: CuaToolResult;
|
||||
try {
|
||||
result = asToolResult(await session.client.callTool({ name, arguments: args }));
|
||||
} catch (error) {
|
||||
if (this.session === session) {
|
||||
this.session = undefined;
|
||||
}
|
||||
await this.closeSession(session);
|
||||
throw error;
|
||||
}
|
||||
if (!result.isError) {
|
||||
return result;
|
||||
}
|
||||
const text = firstTextBlock(result.content);
|
||||
const code = result.structuredContent?.code;
|
||||
if (typeof code === "string") {
|
||||
throw new Error(`COMPUTER_REFUSED_${code}: ${text}`);
|
||||
}
|
||||
throw new Error(`COMPUTER_DRIVER_ERROR: ${text}`);
|
||||
}
|
||||
|
||||
async dispose(): Promise<void> {
|
||||
this.disposed = true;
|
||||
const session = this.session;
|
||||
const pending = this.connectPromise;
|
||||
this.session = undefined;
|
||||
await this.closeSession(session);
|
||||
if (pending) {
|
||||
const pendingSession = await pending.catch(() => undefined);
|
||||
if (pendingSession && pendingSession !== session) {
|
||||
await this.closeSession(pendingSession);
|
||||
}
|
||||
}
|
||||
// Do not kill the daemon: cua-driver runs one shared machine daemon per
|
||||
// endpoint that other clients may attach to, and its idle-session TTL owns
|
||||
// cleanup. Closing our mcp client already releases our transport session
|
||||
// upstream. Killing it would disconnect unrelated clients.
|
||||
this.serveProcess = undefined;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
export type CuaDesktopGeometry = {
|
||||
platform: string;
|
||||
display: string;
|
||||
screenWidth: number;
|
||||
screenHeight: number;
|
||||
scaleFactor: number;
|
||||
screenshotWidth: number;
|
||||
screenshotHeight: number;
|
||||
};
|
||||
|
||||
export type CuaScreenSize = {
|
||||
width: number;
|
||||
height: number;
|
||||
scaleFactor: number;
|
||||
};
|
||||
|
||||
export type CuaLastFrame = {
|
||||
id: string;
|
||||
nativeWidth: number;
|
||||
nativeHeight: number;
|
||||
deliveredWidth: number;
|
||||
deliveredHeight: number;
|
||||
geometry: CuaScreenSize;
|
||||
};
|
||||
|
||||
export type CuaFrameState = {
|
||||
generation: number;
|
||||
lastFrame?: CuaLastFrame;
|
||||
};
|
||||
|
||||
function staleFrame(message: string): Error {
|
||||
return new Error(`COMPUTER_STALE_FRAME: ${message}; take a new screenshot`);
|
||||
}
|
||||
|
||||
/**
|
||||
* cua-driver exposes only the primary-display label, not a stable display ID.
|
||||
* Bind authorization to connection generation plus the complete live geometry.
|
||||
*/
|
||||
export function issueFrame(
|
||||
state: CuaFrameState,
|
||||
geometry: CuaDesktopGeometry,
|
||||
delivered: { width: number; height: number },
|
||||
): string {
|
||||
const digest = createHash("sha256")
|
||||
.update(
|
||||
JSON.stringify([
|
||||
state.generation,
|
||||
geometry.platform,
|
||||
geometry.display,
|
||||
geometry.screenWidth,
|
||||
geometry.screenHeight,
|
||||
geometry.scaleFactor,
|
||||
geometry.screenshotWidth,
|
||||
geometry.screenshotHeight,
|
||||
]),
|
||||
)
|
||||
.digest("hex");
|
||||
const id = `cua:v1:${digest}`;
|
||||
state.lastFrame = {
|
||||
id,
|
||||
nativeWidth: geometry.screenshotWidth,
|
||||
nativeHeight: geometry.screenshotHeight,
|
||||
deliveredWidth: delivered.width,
|
||||
deliveredHeight: delivered.height,
|
||||
geometry: {
|
||||
width: geometry.screenWidth,
|
||||
height: geometry.screenHeight,
|
||||
scaleFactor: geometry.scaleFactor,
|
||||
},
|
||||
};
|
||||
return id;
|
||||
}
|
||||
|
||||
// Accepted limitation: cua-driver 0.10 exposes no stable display identity, only
|
||||
// "display":"primary" (see get_desktop_state). Verification therefore binds to
|
||||
// connection generation plus full live geometry. The generation counter
|
||||
// invalidates every frame on any daemon/session reconnect, which covers RDP
|
||||
// drops and topology changes that break the MCP transport. The only uncaught
|
||||
// case is a same-geometry primary-display substitution that leaves the
|
||||
// connection intact — a corner case upstream gives us no signal to detect.
|
||||
export function verifyFrame(
|
||||
state: CuaFrameState,
|
||||
echoedId: string | undefined,
|
||||
currentScreenSize: CuaScreenSize,
|
||||
): CuaLastFrame {
|
||||
const frame = state.lastFrame;
|
||||
if (!frame || !echoedId || echoedId !== frame.id) {
|
||||
state.lastFrame = undefined;
|
||||
throw staleFrame("the coordinate frame is missing or no longer current");
|
||||
}
|
||||
const geometryMatches =
|
||||
currentScreenSize.width === frame.geometry.width &&
|
||||
currentScreenSize.height === frame.geometry.height &&
|
||||
currentScreenSize.scaleFactor === frame.geometry.scaleFactor;
|
||||
if (!geometryMatches) {
|
||||
state.lastFrame = undefined;
|
||||
throw staleFrame("the primary display geometry changed");
|
||||
}
|
||||
return frame;
|
||||
}
|
||||
|
||||
export function verifyReferenceWidth(
|
||||
state: CuaFrameState,
|
||||
frame: CuaLastFrame,
|
||||
refWidth: number | undefined,
|
||||
): void {
|
||||
if (refWidth === frame.deliveredWidth) {
|
||||
return;
|
||||
}
|
||||
state.lastFrame = undefined;
|
||||
throw staleFrame("the coordinate reference width changed");
|
||||
}
|
||||
Reference in New Issue
Block a user