mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-15 23:24:03 -06:00
feat(cua-computer): full v2 adapter with background window/element delivery (#123604)
* feat(cua-computer): add full v2 adapter * chore(cua-computer): drop release-owned changelog edit
This commit is contained in:
Peter Steinberger
committed by
GitHub
GitHub
parent
72a8b9b5b4
commit
2af5eca07f
@@ -1,7 +1,12 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { createCuaComputerProvider } from "./commands.js";
|
||||
import {
|
||||
CUA_DRIVER_CONTRACT_FIXTURES,
|
||||
cuaToolResult,
|
||||
} from "./cua-driver-contract.test-fixtures.js";
|
||||
import {
|
||||
ClickButton,
|
||||
EscalationReason,
|
||||
ScrollDirection,
|
||||
type CuaDriverSession,
|
||||
type CuaToolResult,
|
||||
@@ -31,6 +36,7 @@ function result(structured: Record<string, unknown>, image = false): CuaToolResu
|
||||
}
|
||||
|
||||
function driver() {
|
||||
let generation = "execution-1";
|
||||
const getDesktopState = vi.fn(async () => result(geometry, true));
|
||||
const getScreenSize = vi.fn(async () => result({ width: 100, height: 50, scale_factor: 1 }));
|
||||
const click = vi.fn(async () => result({}));
|
||||
@@ -39,11 +45,22 @@ function driver() {
|
||||
const scroll = vi.fn(async () => result({}));
|
||||
const typeText = vi.fn(async () => result({}));
|
||||
const pressKey = vi.fn(async () => result({}));
|
||||
const callTool = vi.fn<CuaDriverSession["callTool"]>(async () => result({}));
|
||||
const escalateScope = vi.fn(async () => ({
|
||||
session: "openclaw-test",
|
||||
captureScope: 2,
|
||||
effectiveScope: 1,
|
||||
desktopUnlocked: true,
|
||||
}));
|
||||
const dispose = vi.fn(async () => {});
|
||||
const session: CuaDriverSession = {
|
||||
generation: "execution-1",
|
||||
get generation() {
|
||||
return generation;
|
||||
},
|
||||
isAvailable: () => true,
|
||||
resetAvailabilityCache: () => {},
|
||||
callTool,
|
||||
escalateScope,
|
||||
getDesktopState,
|
||||
getScreenSize,
|
||||
click,
|
||||
@@ -62,9 +79,14 @@ function driver() {
|
||||
drag,
|
||||
moveCursor,
|
||||
scroll,
|
||||
callTool,
|
||||
escalateScope,
|
||||
dispose,
|
||||
typeText,
|
||||
pressKey,
|
||||
setGeneration: (value: string) => {
|
||||
generation = value;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -79,7 +101,7 @@ async function execution(session: CuaDriverSession) {
|
||||
}
|
||||
|
||||
describe("cua-computer provider", () => {
|
||||
it("advertises only its current foreground coordinate capability", () => {
|
||||
it("advertises the implemented Linux v2 capability", () => {
|
||||
const { session } = driver();
|
||||
const descriptor = createCuaComputerProvider({
|
||||
platform: "linux",
|
||||
@@ -90,7 +112,7 @@ describe("cua-computer provider", () => {
|
||||
provider: {
|
||||
id: "cua-computer",
|
||||
label: "CUA Computer",
|
||||
generation: "cua-computer-coordinate-v1",
|
||||
generation: "cua-computer-v2:execution-1",
|
||||
},
|
||||
actions: [
|
||||
"screenshot",
|
||||
@@ -106,16 +128,35 @@ describe("cua-computer provider", () => {
|
||||
"scroll",
|
||||
"type",
|
||||
"key",
|
||||
"hold_key",
|
||||
"wait",
|
||||
"list_apps",
|
||||
"list_windows",
|
||||
"get_accessibility_tree",
|
||||
"get_cursor_position",
|
||||
"get_window_state",
|
||||
"launch_app",
|
||||
"kill_app",
|
||||
"bring_to_front",
|
||||
"set_value",
|
||||
"zoom",
|
||||
"escalate_scope",
|
||||
"invoke_menu",
|
||||
],
|
||||
targets: ["screen"],
|
||||
deliveryModes: ["foreground"],
|
||||
observations: ["image"],
|
||||
targets: ["screen", "window", "element"],
|
||||
deliveryModes: ["background", "foreground"],
|
||||
observations: ["image", "accessibility"],
|
||||
features: { recording: false, agentCursor: false, multiDisplay: false },
|
||||
});
|
||||
});
|
||||
|
||||
it("omits Linux-only held-button actions on Windows", () => {
|
||||
const { session } = driver();
|
||||
const actions = createCuaComputerProvider({ platform: "win32", driver: session }).capabilities()
|
||||
.actions;
|
||||
expect(actions).not.toContain("left_mouse_down");
|
||||
expect(actions).not.toContain("left_mouse_up");
|
||||
expect(actions).toContain("get_window_state");
|
||||
});
|
||||
|
||||
it("uses one typed session for snapshot and frame-authorized click", async () => {
|
||||
const { session, getDesktopState, getScreenSize, click } = driver();
|
||||
const computer = await execution(session);
|
||||
@@ -279,4 +320,359 @@ describe("cua-computer provider", () => {
|
||||
await computer.snapshot('{"format":"png","maxWidth":100}', signal);
|
||||
expect(getDesktopState).toHaveBeenCalledWith(signal);
|
||||
});
|
||||
|
||||
it("mints opaque window and element references and maps background evidence", async () => {
|
||||
const { session, callTool } = driver();
|
||||
callTool.mockImplementation(async (name) => {
|
||||
switch (name) {
|
||||
case "list_windows":
|
||||
return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.listWindows);
|
||||
case "get_window_state":
|
||||
return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.windowState, { image: true });
|
||||
case "click":
|
||||
return cuaToolResult(
|
||||
{},
|
||||
{
|
||||
action:
|
||||
CUA_DRIVER_CONTRACT_FIXTURES.confirmedBackgroundAction as unknown as CuaToolResult["action"],
|
||||
},
|
||||
);
|
||||
default:
|
||||
return cuaToolResult({});
|
||||
}
|
||||
});
|
||||
const computer = await execution(session);
|
||||
const listed = JSON.parse(await computer.act('{"action":"list_windows"}')) as {
|
||||
details: { windows: Array<{ windowRef: string }> };
|
||||
};
|
||||
const windowRef = listed.details.windows[0]!.windowRef;
|
||||
expect(windowRef).toMatch(/^cua:v2:window:/);
|
||||
|
||||
const observed = JSON.parse(
|
||||
await computer.act(JSON.stringify({ action: "get_window_state", windowRef })),
|
||||
) as {
|
||||
observation: {
|
||||
observationId: string;
|
||||
elements: Array<{ elementRef: string }>;
|
||||
};
|
||||
};
|
||||
const { observationId } = observed.observation;
|
||||
const elementRef = observed.observation.elements[0]!.elementRef;
|
||||
expect(observationId).toMatch(/^cua:v2:observation:/);
|
||||
expect(elementRef).toMatch(/^cua:v2:element:/);
|
||||
|
||||
const clicked = JSON.parse(
|
||||
await computer.act(
|
||||
JSON.stringify({
|
||||
action: "left_click",
|
||||
windowRef,
|
||||
elementRef,
|
||||
observationId,
|
||||
deliveryMode: "background",
|
||||
}),
|
||||
),
|
||||
) as { effect: string; details: Record<string, unknown> };
|
||||
expect(clicked).toMatchObject({
|
||||
ok: true,
|
||||
effect: "confirmed",
|
||||
details: {
|
||||
route: "accessibility",
|
||||
deliveryMode: "background",
|
||||
deliveredCount: 1,
|
||||
evidence: ["value_readback"],
|
||||
},
|
||||
});
|
||||
expect(callTool).toHaveBeenLastCalledWith(
|
||||
"click",
|
||||
{
|
||||
pid: 4242,
|
||||
window_id: 99,
|
||||
element_token: "native-element-token-7",
|
||||
button: "left",
|
||||
count: 1,
|
||||
delivery_mode: "background",
|
||||
},
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
it("maps window pixels, app lifecycle, menu, zoom, and escalation tools", async () => {
|
||||
const { session, callTool, escalateScope } = driver();
|
||||
callTool.mockImplementation(async (name) => {
|
||||
switch (name) {
|
||||
case "list_apps":
|
||||
return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.listApps);
|
||||
case "list_windows":
|
||||
return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.listWindows);
|
||||
case "get_window_state":
|
||||
return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.windowState, { image: true });
|
||||
case "zoom":
|
||||
return cuaToolResult({ screenshot_width: 300, screenshot_height: 200 }, { image: true });
|
||||
default:
|
||||
return cuaToolResult(
|
||||
{},
|
||||
{
|
||||
action:
|
||||
CUA_DRIVER_CONTRACT_FIXTURES.suspectedNoopAction as unknown as CuaToolResult["action"],
|
||||
},
|
||||
);
|
||||
}
|
||||
});
|
||||
const computer = await execution(session);
|
||||
const apps = JSON.parse(await computer.act('{"action":"list_apps"}')) as {
|
||||
details: { apps: Array<{ app: string }> };
|
||||
};
|
||||
const app = apps.details.apps[0]!.app;
|
||||
const windows = JSON.parse(await computer.act('{"action":"list_windows"}')) as {
|
||||
details: { windows: Array<{ windowRef: string }> };
|
||||
};
|
||||
const windowRef = windows.details.windows[0]!.windowRef;
|
||||
const observed = JSON.parse(
|
||||
await computer.act(JSON.stringify({ action: "get_window_state", windowRef })),
|
||||
) as { observation: { observationId: string } };
|
||||
|
||||
await computer.act(JSON.stringify({ action: "launch_app", app }));
|
||||
await computer.act(JSON.stringify({ action: "kill_app", app }));
|
||||
await computer.act(
|
||||
JSON.stringify({ action: "invoke_menu", windowRef, path: ["File", "Save"] }),
|
||||
);
|
||||
const zoomed = JSON.parse(
|
||||
await computer.act(
|
||||
JSON.stringify({
|
||||
action: "zoom",
|
||||
windowRef,
|
||||
observationId: observed.observation.observationId,
|
||||
x1: 0,
|
||||
y1: 0,
|
||||
x2: 100,
|
||||
y2: 100,
|
||||
}),
|
||||
),
|
||||
) as { observation: { observationId: string } };
|
||||
expect(zoomed.observation.observationId).not.toBe(observed.observation.observationId);
|
||||
await computer.act(
|
||||
JSON.stringify({ action: "escalate_scope", reason: "background_delivery_failed" }),
|
||||
);
|
||||
|
||||
expect(callTool).toHaveBeenCalledWith(
|
||||
"launch_app",
|
||||
{ launch_path: "/usr/bin/editor" },
|
||||
undefined,
|
||||
);
|
||||
expect(callTool).toHaveBeenCalledWith("kill_app", { pid: 4242 }, undefined);
|
||||
expect(callTool).toHaveBeenCalledWith(
|
||||
"invoke_menu",
|
||||
{ pid: 4242, window_id: 99, path: ["File", "Save"] },
|
||||
undefined,
|
||||
);
|
||||
expect(escalateScope).toHaveBeenCalledWith(
|
||||
EscalationReason.BackgroundDeliveryFailed,
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
it("maps the complete Linux window pointer and keyboard family", async () => {
|
||||
const { session, callTool } = driver();
|
||||
callTool.mockImplementation(async (name) => {
|
||||
if (name === "list_windows") {
|
||||
return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.listWindows);
|
||||
}
|
||||
if (name === "get_window_state") {
|
||||
return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.windowState, { image: true });
|
||||
}
|
||||
return cuaToolResult(
|
||||
{},
|
||||
{
|
||||
action:
|
||||
CUA_DRIVER_CONTRACT_FIXTURES.confirmedBackgroundAction as unknown as CuaToolResult["action"],
|
||||
},
|
||||
);
|
||||
});
|
||||
const computer = await execution(session);
|
||||
const listed = JSON.parse(await computer.act('{"action":"list_windows"}')) as {
|
||||
details: { windows: Array<{ windowRef: string }> };
|
||||
};
|
||||
const windowRef = listed.details.windows[0]!.windowRef;
|
||||
const observed = JSON.parse(
|
||||
await computer.act(JSON.stringify({ action: "get_window_state", windowRef })),
|
||||
) as {
|
||||
observation: { observationId: string; elements: Array<{ elementRef: string }> };
|
||||
};
|
||||
const observationId = observed.observation.observationId;
|
||||
const elementRef = observed.observation.elements[0]!.elementRef;
|
||||
const pixelTarget = { windowRef, observationId, x: 20, y: 30 };
|
||||
const cases = [
|
||||
["right_click", "click", { button: "right", count: 1 }],
|
||||
["middle_click", "click", { button: "middle", count: 1 }],
|
||||
["double_click", "click", { button: "left", count: 2 }],
|
||||
["triple_click", "click", { button: "left", count: 3 }],
|
||||
["left_click_drag", "drag", { from_x: 10, from_y: 15, to_x: 20, to_y: 30, duration_ms: 250 }],
|
||||
["left_mouse_down", "mouse_button_down", { x: 20, y: 30, button: "left" }],
|
||||
["left_mouse_up", "mouse_button_up", { x: 20, y: 30 }],
|
||||
["scroll", "scroll", { direction: "down", by: "line", amount: 4 }],
|
||||
["type", "type_text", { text: "hello", element_token: "native-element-token-7" }],
|
||||
["key", "press_key", { key: "enter", modifiers: ["ctrl"] }],
|
||||
] as const;
|
||||
|
||||
for (const [action, tool, expected] of cases) {
|
||||
const actionInput: Record<string, unknown> = {
|
||||
action,
|
||||
...pixelTarget,
|
||||
deliveryMode: action.startsWith("left_mouse_") ? "background" : "foreground",
|
||||
};
|
||||
if (action === "left_click_drag") {
|
||||
actionInput.fromX = 10;
|
||||
actionInput.fromY = 15;
|
||||
actionInput.durationMs = 250;
|
||||
} else if (action === "scroll") {
|
||||
actionInput.scrollDirection = "down";
|
||||
actionInput.scrollAmount = 4;
|
||||
} else if (action === "type") {
|
||||
actionInput.elementRef = elementRef;
|
||||
actionInput.text = "hello";
|
||||
delete actionInput.x;
|
||||
delete actionInput.y;
|
||||
} else if (action === "key") {
|
||||
actionInput.keys = "ctrl+enter";
|
||||
delete actionInput.x;
|
||||
delete actionInput.y;
|
||||
}
|
||||
await computer.act(JSON.stringify(actionInput));
|
||||
expect(callTool).toHaveBeenCalledWith(
|
||||
tool,
|
||||
expect.objectContaining({ pid: 4242, window_id: 99, ...expected }),
|
||||
undefined,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("maps remaining discovery, window lifecycle, and semantic actions", async () => {
|
||||
const { session, callTool } = driver();
|
||||
callTool.mockImplementation(async (name) => {
|
||||
if (name === "list_windows") {
|
||||
return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.listWindows);
|
||||
}
|
||||
if (name === "get_window_state") {
|
||||
return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.windowState, { image: true });
|
||||
}
|
||||
if (name === "get_accessibility_tree") {
|
||||
return cuaToolResult({
|
||||
processes: [{ pid: 4242, name: "Editor" }],
|
||||
windows: CUA_DRIVER_CONTRACT_FIXTURES.listWindows.windows,
|
||||
});
|
||||
}
|
||||
if (name === "get_cursor_position") {
|
||||
return cuaToolResult({ x: 11, y: 12, source: "x11" });
|
||||
}
|
||||
return cuaToolResult(
|
||||
{},
|
||||
{
|
||||
action:
|
||||
CUA_DRIVER_CONTRACT_FIXTURES.confirmedBackgroundAction as unknown as CuaToolResult["action"],
|
||||
},
|
||||
);
|
||||
});
|
||||
const computer = await execution(session);
|
||||
const tree = JSON.parse(await computer.act('{"action":"get_accessibility_tree"}')) as {
|
||||
details: { windows: unknown[]; processes: unknown[] };
|
||||
};
|
||||
expect(tree.details.windows).toHaveLength(1);
|
||||
expect(tree.details.processes).toHaveLength(1);
|
||||
await expect(computer.act('{"action":"get_cursor_position"}')).resolves.toContain('"x":11');
|
||||
|
||||
const listed = JSON.parse(await computer.act('{"action":"list_windows"}')) as {
|
||||
details: { windows: Array<{ windowRef: string }> };
|
||||
};
|
||||
const windowRef = listed.details.windows[0]!.windowRef;
|
||||
const observed = JSON.parse(
|
||||
await computer.act(JSON.stringify({ action: "get_window_state", windowRef })),
|
||||
) as {
|
||||
observation: { observationId: string; elements: Array<{ elementRef: string }> };
|
||||
};
|
||||
await computer.act(JSON.stringify({ action: "bring_to_front", windowRef }));
|
||||
await computer.act(
|
||||
JSON.stringify({
|
||||
action: "set_value",
|
||||
windowRef,
|
||||
observationId: observed.observation.observationId,
|
||||
elementRef: observed.observation.elements[0]!.elementRef,
|
||||
value: "new",
|
||||
deliveryMode: "background",
|
||||
}),
|
||||
);
|
||||
expect(callTool).toHaveBeenCalledWith(
|
||||
"bring_to_front",
|
||||
{ pid: 4242, window_id: 99 },
|
||||
undefined,
|
||||
);
|
||||
expect(callTool).toHaveBeenCalledWith(
|
||||
"set_value",
|
||||
{
|
||||
pid: 4242,
|
||||
window_id: 99,
|
||||
element_token: "native-element-token-7",
|
||||
value: "new",
|
||||
},
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
it("maps window delivery refusals to the closed computer error prefix", async () => {
|
||||
const { session, callTool } = driver();
|
||||
callTool.mockImplementation(async (name) => {
|
||||
if (name === "list_windows") {
|
||||
return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.listWindows);
|
||||
}
|
||||
if (name === "get_window_state") {
|
||||
return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.windowState, { image: true });
|
||||
}
|
||||
return cuaToolResult(
|
||||
{ code: "background_occluded" },
|
||||
{
|
||||
isError: true,
|
||||
errorCode: "background_occluded",
|
||||
text: "target is occluded",
|
||||
},
|
||||
);
|
||||
});
|
||||
const computer = await execution(session);
|
||||
const listed = JSON.parse(await computer.act('{"action":"list_windows"}')) as {
|
||||
details: { windows: Array<{ windowRef: string }> };
|
||||
};
|
||||
const windowRef = listed.details.windows[0]!.windowRef;
|
||||
const observed = JSON.parse(
|
||||
await computer.act(JSON.stringify({ action: "get_window_state", windowRef })),
|
||||
) as { observation: { observationId: string } };
|
||||
await expect(
|
||||
computer.act(
|
||||
JSON.stringify({
|
||||
action: "left_click",
|
||||
windowRef,
|
||||
observationId: observed.observation.observationId,
|
||||
x: 10,
|
||||
y: 20,
|
||||
}),
|
||||
),
|
||||
).rejects.toThrow("COMPUTER_REFUSED_background_occluded");
|
||||
});
|
||||
|
||||
it("invalidates observation references when the driver generation rotates", async () => {
|
||||
const { session, callTool, setGeneration } = driver();
|
||||
callTool.mockImplementation(async (name) =>
|
||||
name === "list_windows"
|
||||
? cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.listWindows)
|
||||
: cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.windowState, { image: true }),
|
||||
);
|
||||
const computer = await execution(session);
|
||||
const listed = JSON.parse(await computer.act('{"action":"list_windows"}')) as {
|
||||
details: { windows: Array<{ windowRef: string }> };
|
||||
};
|
||||
const windowRef = listed.details.windows[0]!.windowRef;
|
||||
setGeneration("execution-2");
|
||||
|
||||
await expect(
|
||||
computer.act(JSON.stringify({ action: "get_window_state", windowRef })),
|
||||
).rejects.toThrow("COMPUTER_STALE_OBSERVATION");
|
||||
expect(callTool).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -19,7 +19,9 @@ import {
|
||||
type CuaDriverSession,
|
||||
type CuaToolResult,
|
||||
} from "./driver-client.js";
|
||||
import { platformActions } from "./driver-result.js";
|
||||
import {
|
||||
adoptGeneration,
|
||||
issueFrame,
|
||||
verifyFrame,
|
||||
verifyReferenceWidth,
|
||||
@@ -28,26 +30,10 @@ import {
|
||||
type CuaLastFrame,
|
||||
type CuaScreenSize,
|
||||
} from "./frame.js";
|
||||
import { handleV2Act, type CuaComputerActParams } from "./v2-actions.js";
|
||||
|
||||
const AVAILABILITY_POLL_MS = 5_000;
|
||||
const CUA_COORDINATE_ACTION_NAMES = COMPUTER_USE_V2_ACTION_NAMES.slice(0, 15);
|
||||
const CUA_WIRE_ACTION_NAMES = COMPUTER_USE_V2_ACTION_NAMES.slice(1, 14);
|
||||
type CuaComputerActParams = {
|
||||
action: ComputerActParams["action"];
|
||||
displayFrameId?: string;
|
||||
x?: number;
|
||||
y?: number;
|
||||
fromX?: number;
|
||||
fromY?: number;
|
||||
text?: string;
|
||||
keys?: string;
|
||||
modifiers?: string;
|
||||
scrollDirection?: "up" | "down" | "left" | "right";
|
||||
scrollAmount?: number;
|
||||
durationMs?: number;
|
||||
screenIndex?: number;
|
||||
refWidth?: number;
|
||||
};
|
||||
// Rastermill enforces inputPixels before resizing, so this must clear the native
|
||||
// capture, not the delivered frame. 8K (7680x4320 = ~33.2M) is a valid primary
|
||||
// display; budget above it so full-resolution snapshots reach the downscaler.
|
||||
@@ -244,7 +230,7 @@ async function currentFrame(
|
||||
return frame;
|
||||
}
|
||||
|
||||
async function handleAct(
|
||||
async function handleDesktopAct(
|
||||
driver: CuaDriverSession,
|
||||
frameState: CuaFrameState,
|
||||
params: ComputerActParams,
|
||||
@@ -441,12 +427,14 @@ export function createCuaComputerProvider(
|
||||
provider: {
|
||||
id: "cua-computer",
|
||||
label: "CUA Computer",
|
||||
generation: "cua-computer-coordinate-v1",
|
||||
generation: isSupportedPlatform
|
||||
? `cua-computer-v2:${driver().generation}`
|
||||
: "cua-computer-v2:unsupported",
|
||||
},
|
||||
actions: CUA_COORDINATE_ACTION_NAMES,
|
||||
targets: ["screen"],
|
||||
deliveryModes: ["foreground"],
|
||||
observations: ["image"],
|
||||
actions: platformActions(platform),
|
||||
targets: ["screen", "window", "element"],
|
||||
deliveryModes: ["background", "foreground"],
|
||||
observations: ["image", "accessibility"],
|
||||
features: { recording: false, agentCursor: false, multiDisplay: false },
|
||||
}),
|
||||
isAvailable,
|
||||
@@ -468,7 +456,7 @@ export function createCuaComputerProvider(
|
||||
},
|
||||
openExecution: async () => {
|
||||
const queue = new PromiseQueue();
|
||||
const frameState: CuaFrameState = { generation: "uninitialized" };
|
||||
const frameState: CuaFrameState = { generation: driver().generation };
|
||||
return {
|
||||
snapshot: async (paramsJSON, signal) =>
|
||||
await queue.run(async () => {
|
||||
@@ -511,7 +499,7 @@ export function createCuaComputerProvider(
|
||||
width = result.width;
|
||||
height = result.height;
|
||||
}
|
||||
frameState.generation = driver().generation;
|
||||
adoptGeneration(frameState, driver().generation);
|
||||
const displayFrameId = issueFrame(frameState, geometry, { width, height });
|
||||
return JSON.stringify({
|
||||
format,
|
||||
@@ -529,10 +517,12 @@ export function createCuaComputerProvider(
|
||||
"COMPUTER_DRIVER_UNAVAILABLE: cua-computer supports Windows and Linux",
|
||||
);
|
||||
}
|
||||
return await handleAct(
|
||||
return await handleV2Act(
|
||||
platform,
|
||||
driver(),
|
||||
frameState,
|
||||
parseComputerActParamsJSON(paramsJSON),
|
||||
handleDesktopAct,
|
||||
signal,
|
||||
);
|
||||
}),
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
import type { CuaToolResult } from "./driver-client.js";
|
||||
|
||||
export const CUA_DRIVER_CONTRACT_FIXTURES = {
|
||||
listApps: {
|
||||
apps: [
|
||||
{
|
||||
pid: 4242,
|
||||
bundle_id: "org.example.Editor",
|
||||
name: "Editor",
|
||||
running: true,
|
||||
active: false,
|
||||
kind: "desktop",
|
||||
launch_path: "/usr/bin/editor",
|
||||
last_used: "2026-08-14T00:00:00Z",
|
||||
},
|
||||
],
|
||||
},
|
||||
listWindows: {
|
||||
windows: [
|
||||
{
|
||||
window_id: 99,
|
||||
pid: 4242,
|
||||
app_name: "Editor",
|
||||
title: "Notes",
|
||||
bounds: { x: 40, y: 50, width: 800, height: 600 },
|
||||
is_on_screen: true,
|
||||
minimized: false,
|
||||
z_index: 2,
|
||||
},
|
||||
],
|
||||
},
|
||||
windowState: {
|
||||
window_id: 99,
|
||||
pid: 4242,
|
||||
snapshot_id: "native-snapshot-1",
|
||||
total_element_count: 1,
|
||||
returned_element_count: 1,
|
||||
screenshot_width: 800,
|
||||
screenshot_height: 600,
|
||||
screenshot_mime_type: "image/png",
|
||||
elements: [
|
||||
{
|
||||
element_index: 7,
|
||||
element_token: "native-element-token-7",
|
||||
role: "text field",
|
||||
label: "Body",
|
||||
value: "old",
|
||||
frame: { x: 80, y: 100, w: 400, h: 240 },
|
||||
},
|
||||
],
|
||||
},
|
||||
confirmedBackgroundAction: {
|
||||
effect: 0,
|
||||
route: 0,
|
||||
delivery: { mode: 0, deliveredCount: 1 },
|
||||
evidence: [{ kind: 0 }],
|
||||
},
|
||||
suspectedNoopAction: {
|
||||
effect: 3,
|
||||
route: 1,
|
||||
delivery: { mode: 0 },
|
||||
escalation: { target: 1, reason: 3 },
|
||||
},
|
||||
} as const;
|
||||
|
||||
export function cuaToolResult(
|
||||
structured: Record<string, unknown>,
|
||||
options: {
|
||||
action?: CuaToolResult["action"];
|
||||
image?: boolean;
|
||||
isError?: boolean;
|
||||
errorCode?: string;
|
||||
text?: string;
|
||||
} = {},
|
||||
): CuaToolResult {
|
||||
return {
|
||||
text: options.text ?? "ok",
|
||||
images: options.image
|
||||
? [{ mimeType: "image/png", dataBase64: Buffer.from("png").toString("base64") }]
|
||||
: [],
|
||||
structuredJson: JSON.stringify(structured),
|
||||
isError: options.isError ?? false,
|
||||
...(options.errorCode ? { errorCode: options.errorCode } : {}),
|
||||
...(options.action ? { action: options.action } : {}),
|
||||
degraded: false,
|
||||
rawJson: "{}",
|
||||
};
|
||||
}
|
||||
@@ -2,10 +2,17 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
close: vi.fn(),
|
||||
callTool: vi.fn(async () => ({})),
|
||||
create: vi.fn(),
|
||||
createConfigured: vi.fn(),
|
||||
createTrustedSession: vi.fn(),
|
||||
endSession: vi.fn(async () => ({})),
|
||||
escalateSession: vi.fn(async () => ({
|
||||
session: "openclaw-test",
|
||||
captureScope: "desktop",
|
||||
effectiveScope: "desktop",
|
||||
desktopUnlocked: true,
|
||||
})),
|
||||
getDesktopState: vi.fn(async () => ({})),
|
||||
isAvailable: vi.fn(() => true),
|
||||
startSession: vi.fn(async () => ({})),
|
||||
@@ -13,10 +20,11 @@ const mocks = vi.hoisted(() => ({
|
||||
}));
|
||||
|
||||
const sdk = {
|
||||
CaptureScope: { Desktop: "desktop" },
|
||||
CaptureScope: { Window: "window", Desktop: "desktop" },
|
||||
ClickButton: { Left: 0, Right: 1, Middle: 2 },
|
||||
CuaDriver: { create: mocks.create, createConfigured: mocks.createConfigured },
|
||||
DesktopScope: { Desktop: 0 },
|
||||
EscalationReason: { Other: "other" },
|
||||
ScrollBy: { Line: 0 },
|
||||
ScrollDirection: { Up: 0, Down: 1, Left: 2, Right: 3 },
|
||||
SessionPermissionMode: { Unrestricted: "unrestricted" },
|
||||
@@ -42,7 +50,9 @@ describe("CUA Driver direct session", () => {
|
||||
});
|
||||
mocks.createTrustedSession.mockReturnValue({
|
||||
close: mocks.close,
|
||||
callTool: mocks.callTool,
|
||||
endSession: mocks.endSession,
|
||||
escalateSession: mocks.escalateSession,
|
||||
getDesktopState: mocks.getDesktopState,
|
||||
startSession: mocks.startSession,
|
||||
});
|
||||
@@ -109,6 +119,33 @@ describe("CUA Driver direct session", () => {
|
||||
expect(mocks.endSession).toHaveBeenCalledWith({ session: sessionOptions.publicSession });
|
||||
});
|
||||
|
||||
it("starts window-scoped generic tools and widens only for an explicit desktop call", async () => {
|
||||
const driver = createCuaDriver({ loadSdk: () => sdk as never });
|
||||
|
||||
await driver.callTool("list_windows", {});
|
||||
const sessionOptions = mocks.createTrustedSession.mock.calls[0]?.[1];
|
||||
expect(mocks.startSession).toHaveBeenCalledWith(
|
||||
{ session: sessionOptions.publicSession, captureScope: "window" },
|
||||
undefined,
|
||||
);
|
||||
expect(mocks.callTool).toHaveBeenCalledWith(
|
||||
"list_windows",
|
||||
JSON.stringify({ session: sessionOptions.publicSession }),
|
||||
undefined,
|
||||
);
|
||||
|
||||
await driver.getDesktopState();
|
||||
expect(mocks.escalateSession).toHaveBeenCalledWith(
|
||||
{
|
||||
session: sessionOptions.publicSession,
|
||||
reason: "other",
|
||||
detail: "explicit desktop-scope OpenClaw action",
|
||||
},
|
||||
undefined,
|
||||
);
|
||||
await driver.dispose();
|
||||
});
|
||||
|
||||
it("keeps a missing native desktop library behind command availability", async () => {
|
||||
const loadSdk = vi.fn(() => {
|
||||
throw new Error("libX11.so.6: cannot open shared object file");
|
||||
|
||||
@@ -1,14 +1,18 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
|
||||
type DriverClickButton = import("@trycua/cua-driver").ClickButton;
|
||||
type DriverCaptureScope = import("@trycua/cua-driver").CaptureScope;
|
||||
type DriverEscalationReason = import("@trycua/cua-driver").EscalationReason;
|
||||
type CuaDriverLike = import("@trycua/cua-driver").CuaDriverLike;
|
||||
type CuaDriverSessionLike = import("@trycua/cua-driver").CuaDriverSessionLike;
|
||||
type DriverScrollDirection = import("@trycua/cua-driver").ScrollDirection;
|
||||
type CuaSessionState = import("@trycua/cua-driver").SessionStateOutput;
|
||||
type CuaDriverSdk = Pick<
|
||||
typeof import("@trycua/cua-driver"),
|
||||
| "CaptureScope"
|
||||
| "CuaDriver"
|
||||
| "DesktopScope"
|
||||
| "EscalationReason"
|
||||
| "ScrollBy"
|
||||
| "SessionPermissionMode"
|
||||
| "createTrustedSession"
|
||||
@@ -16,6 +20,15 @@ type CuaDriverSdk = Pick<
|
||||
|
||||
export type CuaToolResult = import("@trycua/cua-driver").ToolResult;
|
||||
|
||||
export const EscalationReason = {
|
||||
AxTreePixelMismatch: 0 as DriverEscalationReason,
|
||||
BackgroundDeliveryFailed: 1 as DriverEscalationReason,
|
||||
ForegroundIneffective: 2 as DriverEscalationReason,
|
||||
NoWindowTarget: 3 as DriverEscalationReason,
|
||||
Other: 4 as DriverEscalationReason,
|
||||
} as const;
|
||||
export type EscalationReason = (typeof EscalationReason)[keyof typeof EscalationReason];
|
||||
|
||||
// These numeric values are part of the pinned 0.19.3 SDK contract. Keeping
|
||||
// them local avoids loading the native library while OpenClaw is only
|
||||
// registering the bundled plugin.
|
||||
@@ -38,6 +51,12 @@ export interface CuaDriverSession {
|
||||
readonly generation: string;
|
||||
isAvailable(): boolean;
|
||||
resetAvailabilityCache(): void;
|
||||
callTool(
|
||||
name: string,
|
||||
args: Record<string, unknown>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<CuaToolResult>;
|
||||
escalateScope(reason: EscalationReason, signal?: AbortSignal): Promise<CuaSessionState>;
|
||||
getDesktopState(signal?: AbortSignal): Promise<CuaToolResult>;
|
||||
getScreenSize(signal?: AbortSignal): Promise<CuaToolResult>;
|
||||
click(
|
||||
@@ -71,6 +90,8 @@ class DirectCuaDriverSession implements CuaDriverSession {
|
||||
private readonly session: CuaDriverSessionLike;
|
||||
private readonly publicSession = `openclaw-${randomUUID()}`;
|
||||
private startPromise: Promise<void> | undefined;
|
||||
private desktopEscalationPromise: Promise<void> | undefined;
|
||||
private captureScope: DriverCaptureScope | undefined;
|
||||
private started = false;
|
||||
private disposed = false;
|
||||
|
||||
@@ -99,16 +120,17 @@ class DirectCuaDriverSession implements CuaDriverSession {
|
||||
});
|
||||
}
|
||||
|
||||
private async ensureStarted(signal?: AbortSignal): Promise<void> {
|
||||
private async ensureStarted(
|
||||
captureScope: DriverCaptureScope,
|
||||
signal?: AbortSignal,
|
||||
): Promise<void> {
|
||||
if (this.disposed) {
|
||||
throw new Error("COMPUTER_DRIVER_UNAVAILABLE: cua-computer is stopping");
|
||||
}
|
||||
if (!this.startPromise) {
|
||||
this.captureScope = captureScope;
|
||||
const start = this.session
|
||||
.startSession(
|
||||
{ session: this.publicSession, captureScope: this.sdk.CaptureScope.Desktop },
|
||||
asyncOptions(signal),
|
||||
)
|
||||
.startSession({ session: this.publicSession, captureScope }, asyncOptions(signal))
|
||||
.then(() => {
|
||||
this.started = true;
|
||||
});
|
||||
@@ -124,13 +146,38 @@ class DirectCuaDriverSession implements CuaDriverSession {
|
||||
return;
|
||||
}
|
||||
await this.startPromise;
|
||||
if (
|
||||
captureScope === this.sdk.CaptureScope.Desktop &&
|
||||
this.captureScope !== this.sdk.CaptureScope.Desktop
|
||||
) {
|
||||
await this.ensureDesktopScope(signal);
|
||||
}
|
||||
}
|
||||
|
||||
private async ensureDesktopScope(signal?: AbortSignal): Promise<void> {
|
||||
if (!this.desktopEscalationPromise) {
|
||||
this.desktopEscalationPromise = this.session
|
||||
.escalateSession(
|
||||
{
|
||||
session: this.publicSession,
|
||||
reason: this.sdk.EscalationReason.Other,
|
||||
detail: "explicit desktop-scope OpenClaw action",
|
||||
},
|
||||
asyncOptions(signal),
|
||||
)
|
||||
.then(() => {
|
||||
this.captureScope = this.sdk.CaptureScope.Desktop;
|
||||
});
|
||||
}
|
||||
await this.desktopEscalationPromise;
|
||||
}
|
||||
|
||||
private async invoke<T>(
|
||||
captureScope: DriverCaptureScope,
|
||||
signal: AbortSignal | undefined,
|
||||
operation: () => Promise<T>,
|
||||
): Promise<T> {
|
||||
await this.ensureStarted(signal);
|
||||
await this.ensureStarted(captureScope, signal);
|
||||
return await operation();
|
||||
}
|
||||
|
||||
@@ -138,17 +185,39 @@ class DirectCuaDriverSession implements CuaDriverSession {
|
||||
return !this.disposed && this.runtime.isAvailable();
|
||||
}
|
||||
resetAvailabilityCache(): void {}
|
||||
async callTool(name: string, args: Record<string, unknown>, signal?: AbortSignal) {
|
||||
return await this.invoke(this.sdk.CaptureScope.Window, signal, () =>
|
||||
this.session.callTool(
|
||||
name,
|
||||
JSON.stringify({ ...args, session: this.publicSession }),
|
||||
asyncOptions(signal),
|
||||
),
|
||||
);
|
||||
}
|
||||
async escalateScope(reason: EscalationReason, signal?: AbortSignal) {
|
||||
await this.ensureStarted(this.sdk.CaptureScope.Window, signal);
|
||||
const state = await this.session.escalateSession(
|
||||
{ session: this.publicSession, reason },
|
||||
asyncOptions(signal),
|
||||
);
|
||||
this.captureScope = this.sdk.CaptureScope.Desktop;
|
||||
return state;
|
||||
}
|
||||
async getDesktopState(signal?: AbortSignal) {
|
||||
return await this.invoke(signal, () => this.session.getDesktopState({}, asyncOptions(signal)));
|
||||
return await this.invoke(this.sdk.CaptureScope.Desktop, signal, () =>
|
||||
this.session.getDesktopState({}, asyncOptions(signal)),
|
||||
);
|
||||
}
|
||||
async getScreenSize(signal?: AbortSignal) {
|
||||
return await this.invoke(signal, () => this.session.getScreenSize({}, asyncOptions(signal)));
|
||||
return await this.invoke(this.sdk.CaptureScope.Desktop, signal, () =>
|
||||
this.session.getScreenSize({}, asyncOptions(signal)),
|
||||
);
|
||||
}
|
||||
async click(
|
||||
input: { x: number; y: number; button: ClickButton; count: number },
|
||||
signal?: AbortSignal,
|
||||
) {
|
||||
return await this.invoke(signal, () =>
|
||||
return await this.invoke(this.sdk.CaptureScope.Desktop, signal, () =>
|
||||
this.session.click({ ...input, scope: this.sdk.DesktopScope.Desktop }, asyncOptions(signal)),
|
||||
);
|
||||
}
|
||||
@@ -156,12 +225,12 @@ class DirectCuaDriverSession implements CuaDriverSession {
|
||||
input: { fromX: number; fromY: number; toX: number; toY: number; durationMs?: bigint },
|
||||
signal?: AbortSignal,
|
||||
) {
|
||||
return await this.invoke(signal, () =>
|
||||
return await this.invoke(this.sdk.CaptureScope.Desktop, signal, () =>
|
||||
this.session.drag({ ...input, scope: this.sdk.DesktopScope.Desktop }, asyncOptions(signal)),
|
||||
);
|
||||
}
|
||||
async moveCursor(input: { x: number; y: number }, signal?: AbortSignal) {
|
||||
return await this.invoke(signal, () =>
|
||||
return await this.invoke(this.sdk.CaptureScope.Desktop, signal, () =>
|
||||
this.session.moveCursor(
|
||||
{ ...input, scope: this.sdk.DesktopScope.Desktop },
|
||||
asyncOptions(signal),
|
||||
@@ -172,7 +241,7 @@ class DirectCuaDriverSession implements CuaDriverSession {
|
||||
input: { x: number; y: number; direction: ScrollDirection; amount: bigint },
|
||||
signal?: AbortSignal,
|
||||
) {
|
||||
return await this.invoke(signal, () =>
|
||||
return await this.invoke(this.sdk.CaptureScope.Desktop, signal, () =>
|
||||
this.session.scroll(
|
||||
{
|
||||
...input,
|
||||
@@ -184,12 +253,12 @@ class DirectCuaDriverSession implements CuaDriverSession {
|
||||
);
|
||||
}
|
||||
async typeText(text: string, signal?: AbortSignal) {
|
||||
return await this.invoke(signal, () =>
|
||||
return await this.invoke(this.sdk.CaptureScope.Desktop, signal, () =>
|
||||
this.session.typeText({ text, scope: this.sdk.DesktopScope.Desktop }, asyncOptions(signal)),
|
||||
);
|
||||
}
|
||||
async pressKey(input: { key: string; modifiers: string[] }, signal?: AbortSignal) {
|
||||
return await this.invoke(signal, () =>
|
||||
return await this.invoke(this.sdk.CaptureScope.Desktop, signal, () =>
|
||||
this.session.pressKey(
|
||||
{ ...input, scope: this.sdk.DesktopScope.Desktop },
|
||||
asyncOptions(signal),
|
||||
@@ -344,6 +413,12 @@ class LazyCuaDriverSession implements CuaDriverSession {
|
||||
async getDesktopState(signal?: AbortSignal) {
|
||||
return await (await this.requireRuntime()).getDesktopState(signal);
|
||||
}
|
||||
async callTool(name: string, args: Record<string, unknown>, signal?: AbortSignal) {
|
||||
return await (await this.requireRuntime()).callTool(name, args, signal);
|
||||
}
|
||||
async escalateScope(reason: EscalationReason, signal?: AbortSignal) {
|
||||
return await (await this.requireRuntime()).escalateScope(reason, signal);
|
||||
}
|
||||
async getScreenSize(signal?: AbortSignal) {
|
||||
return await (await this.requireRuntime()).getScreenSize(signal);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,364 @@
|
||||
import {
|
||||
COMPUTER_USE_V2_ACTION_NAMES,
|
||||
type ComputerActResult,
|
||||
type ComputerUseV2ActionName,
|
||||
} from "openclaw/plugin-sdk/computer-use";
|
||||
import { canonicalizeBase64 } from "openclaw/plugin-sdk/media-runtime";
|
||||
import { z } from "zod";
|
||||
import type { CuaDriverSession, CuaToolResult } from "./driver-client.js";
|
||||
import {
|
||||
adoptGeneration,
|
||||
issueAppRef,
|
||||
issueElementRef,
|
||||
issueObservation,
|
||||
issueWindowRef,
|
||||
type CuaFrameState,
|
||||
} from "./frame.js";
|
||||
|
||||
const CUA_WIRE_ACTION_NAMES = COMPUTER_USE_V2_ACTION_NAMES.slice(1, 14);
|
||||
const CUA_COMMON_ACTION_NAMES = [
|
||||
"screenshot",
|
||||
...CUA_WIRE_ACTION_NAMES.filter((action) => action !== "hold_key"),
|
||||
"list_apps",
|
||||
"list_windows",
|
||||
"get_accessibility_tree",
|
||||
"get_cursor_position",
|
||||
"get_window_state",
|
||||
"launch_app",
|
||||
"kill_app",
|
||||
"bring_to_front",
|
||||
"set_value",
|
||||
"zoom",
|
||||
"escalate_scope",
|
||||
"invoke_menu",
|
||||
] as const;
|
||||
|
||||
const NativeAppSchema = z.object({
|
||||
pid: z.number().int().nonnegative().nullable().optional(),
|
||||
bundle_id: z.string().nullable().optional(),
|
||||
name: z.string().min(1),
|
||||
running: z.boolean().nullable().optional(),
|
||||
active: z.boolean().optional(),
|
||||
kind: z.string().nullable().optional(),
|
||||
launch_path: z.string().nullable().optional(),
|
||||
last_used: z.string().nullable().optional(),
|
||||
});
|
||||
const NativeBoundsSchema = z.object({
|
||||
x: z.number(),
|
||||
y: z.number(),
|
||||
width: z.number().nonnegative(),
|
||||
height: z.number().nonnegative(),
|
||||
});
|
||||
const NativeWindowSchema = z.object({
|
||||
window_id: z.number().int().nonnegative(),
|
||||
pid: z.number().int().positive().nullable().optional(),
|
||||
app_name: z.string().optional(),
|
||||
title: z.string().optional(),
|
||||
bounds: NativeBoundsSchema,
|
||||
is_on_screen: z.boolean().optional(),
|
||||
minimized: z.boolean().optional(),
|
||||
z_index: z.number().int().nullable().optional(),
|
||||
});
|
||||
const NativeElementSchema = z.object({
|
||||
element_index: z.number().int().nonnegative(),
|
||||
element_token: z.string().min(1).optional(),
|
||||
role: z.string().optional(),
|
||||
label: z.string().optional(),
|
||||
value: z.string().optional(),
|
||||
frame: z
|
||||
.object({
|
||||
x: z.number(),
|
||||
y: z.number(),
|
||||
w: z.number().nonnegative(),
|
||||
h: z.number().nonnegative(),
|
||||
})
|
||||
.optional(),
|
||||
});
|
||||
const MAX_DISCOVERY_ITEMS = 500;
|
||||
const PARTIAL_EFFECT = 1 as import("@trycua/cua-driver").ActionEffect;
|
||||
const VALUE_READBACK_EVIDENCE = 0 as import("@trycua/cua-driver").ActionEvidenceKind;
|
||||
|
||||
export function platformActions(platform: NodeJS.Platform): ComputerUseV2ActionName[] {
|
||||
return CUA_COMMON_ACTION_NAMES.filter(
|
||||
(action) =>
|
||||
platform === "linux" || (action !== "left_mouse_down" && action !== "left_mouse_up"),
|
||||
) as ComputerUseV2ActionName[];
|
||||
}
|
||||
|
||||
function boundedItems<T>(items: T[]): { items: T[]; truncated: number } {
|
||||
return {
|
||||
items: items.slice(0, MAX_DISCOVERY_ITEMS),
|
||||
truncated: Math.max(0, items.length - MAX_DISCOVERY_ITEMS),
|
||||
};
|
||||
}
|
||||
|
||||
function driverEffect(result: CuaToolResult): ComputerActResult["effect"] | undefined {
|
||||
switch (Number(result.action?.effect)) {
|
||||
case 0:
|
||||
return "confirmed";
|
||||
case 1:
|
||||
case 2:
|
||||
return "unverifiable";
|
||||
case 3:
|
||||
return "suspected_noop";
|
||||
case 4:
|
||||
throw new Error("COMPUTER_REFUSED_action_refused: CUA Driver refused the action");
|
||||
default:
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function driverEscalation(result: CuaToolResult): ComputerActResult["escalation"] | undefined {
|
||||
const escalation = result.action?.escalation;
|
||||
if (!escalation) {
|
||||
return undefined;
|
||||
}
|
||||
const recommended = {
|
||||
0: "window-pixel",
|
||||
1: "foreground",
|
||||
2: "window-pixel",
|
||||
3: "desktop",
|
||||
}[escalation.target] as NonNullable<ComputerActResult["escalation"]>["recommended"] | undefined;
|
||||
const reasonCode = {
|
||||
0: "route_unavailable",
|
||||
1: "delivery_failed",
|
||||
2: "effect_unconfirmed",
|
||||
3: "suspected_noop",
|
||||
4: "permission_required",
|
||||
}[escalation.reason];
|
||||
if (!recommended || !reasonCode) {
|
||||
throw new Error("COMPUTER_DRIVER_ERROR: invalid CUA Driver action escalation");
|
||||
}
|
||||
return { recommended, reasonCode };
|
||||
}
|
||||
|
||||
function driverActionDetails(result: CuaToolResult): Record<string, unknown> | undefined {
|
||||
const action = result.action;
|
||||
if (!action) {
|
||||
return undefined;
|
||||
}
|
||||
const details: Record<string, unknown> = {
|
||||
route: [
|
||||
"accessibility",
|
||||
"synthetic_events",
|
||||
"global_input",
|
||||
"system_api",
|
||||
"dom",
|
||||
"trusted_input",
|
||||
][action.route],
|
||||
};
|
||||
if (action.effect === PARTIAL_EFFECT) {
|
||||
details.partial = true;
|
||||
}
|
||||
if (action.delivery) {
|
||||
details.deliveryMode = ["background", "foreground", "not_applicable", "unknown"][
|
||||
action.delivery.mode
|
||||
];
|
||||
if (action.delivery.deliveredCount !== undefined) {
|
||||
details.deliveredCount = action.delivery.deliveredCount;
|
||||
}
|
||||
}
|
||||
if (action.evidence?.length) {
|
||||
details.evidence = action.evidence.map(({ kind }) =>
|
||||
kind === VALUE_READBACK_EVIDENCE ? "value_readback" : "window_change",
|
||||
);
|
||||
}
|
||||
return Object.values(details).some((value) => value !== undefined) ? details : undefined;
|
||||
}
|
||||
|
||||
export function actionEnvelope(
|
||||
result: CuaToolResult,
|
||||
details?: Record<string, unknown>,
|
||||
): ComputerActResult {
|
||||
const effect = driverEffect(result);
|
||||
const escalation = driverEscalation(result);
|
||||
const driverDetails = driverActionDetails(result);
|
||||
return {
|
||||
ok: true,
|
||||
...(effect ? { effect } : {}),
|
||||
...(escalation ? { escalation } : {}),
|
||||
...(driverDetails || details ? { details: { ...driverDetails, ...details } } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export async function callWindowTool(
|
||||
driver: CuaDriverSession,
|
||||
state: CuaFrameState,
|
||||
name: string,
|
||||
args: Record<string, unknown>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<CuaToolResult> {
|
||||
const result = await driver.callTool(name, args, signal);
|
||||
adoptGeneration(state, driver.generation);
|
||||
if (result.isError) {
|
||||
const code = result.errorCode
|
||||
? `COMPUTER_REFUSED_${result.errorCode}`
|
||||
: "COMPUTER_DRIVER_ERROR";
|
||||
throw new Error(`${code}: ${result.text || `${name} failed`}`);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export function projectedToolDetails(result: CuaToolResult, tool: string): Record<string, unknown> {
|
||||
if (!result.structuredJson) {
|
||||
throw new Error(`COMPUTER_DRIVER_ERROR: ${tool} returned no structuredContent`);
|
||||
}
|
||||
try {
|
||||
const value: unknown = JSON.parse(result.structuredJson);
|
||||
if (value && typeof value === "object" && !Array.isArray(value)) {
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
} catch {}
|
||||
throw new Error(`COMPUTER_DRIVER_ERROR: ${tool} returned invalid structuredContent`);
|
||||
}
|
||||
|
||||
export function nativeWindows(value: unknown): Array<z.infer<typeof NativeWindowSchema>> {
|
||||
if (!Array.isArray(value)) {
|
||||
return [];
|
||||
}
|
||||
return value.flatMap((entry) => {
|
||||
const parsed = NativeWindowSchema.safeParse(entry);
|
||||
return parsed.success && parsed.data.pid ? [parsed.data] : [];
|
||||
});
|
||||
}
|
||||
|
||||
export function projectWindows(
|
||||
state: CuaFrameState,
|
||||
windows: Array<z.infer<typeof NativeWindowSchema>>,
|
||||
): { windows: Array<Record<string, unknown>>; truncatedWindows?: number } {
|
||||
const bounded = boundedItems(windows);
|
||||
return {
|
||||
windows: bounded.items.map((window) => ({
|
||||
windowRef: issueWindowRef(state, { pid: window.pid!, windowId: window.window_id }),
|
||||
...(window.app_name ? { appName: window.app_name } : {}),
|
||||
...(window.title ? { title: window.title } : {}),
|
||||
bounds: window.bounds,
|
||||
...(window.is_on_screen !== undefined ? { isOnScreen: window.is_on_screen } : {}),
|
||||
...(window.minimized !== undefined ? { minimized: window.minimized } : {}),
|
||||
...(window.z_index !== undefined ? { zIndex: window.z_index } : {}),
|
||||
})),
|
||||
...(bounded.truncated ? { truncatedWindows: bounded.truncated } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function projectApps(state: CuaFrameState, value: unknown): Record<string, unknown> {
|
||||
const raw = Array.isArray(value) ? value : [];
|
||||
const apps = raw.flatMap((entry) => {
|
||||
const parsed = NativeAppSchema.safeParse(entry);
|
||||
if (!parsed.success) {
|
||||
return [];
|
||||
}
|
||||
const app = issueAppRef(state, {
|
||||
...(parsed.data.pid ? { pid: parsed.data.pid } : {}),
|
||||
name: parsed.data.name,
|
||||
...(parsed.data.bundle_id ? { bundleId: parsed.data.bundle_id } : {}),
|
||||
...(parsed.data.launch_path ? { launchPath: parsed.data.launch_path } : {}),
|
||||
});
|
||||
return [
|
||||
{
|
||||
app,
|
||||
name: parsed.data.name,
|
||||
...(parsed.data.running !== undefined ? { running: parsed.data.running } : {}),
|
||||
...(parsed.data.active !== undefined ? { active: parsed.data.active } : {}),
|
||||
...(parsed.data.kind ? { kind: parsed.data.kind } : {}),
|
||||
...(parsed.data.last_used ? { lastUsed: parsed.data.last_used } : {}),
|
||||
},
|
||||
];
|
||||
});
|
||||
const bounded = boundedItems(apps);
|
||||
return {
|
||||
apps: bounded.items,
|
||||
totalApps: apps.length,
|
||||
...(bounded.truncated ? { truncatedApps: bounded.truncated } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function projectProcesses(value: unknown): Record<string, unknown> {
|
||||
const processes = boundedItems(Array.isArray(value) ? value : []);
|
||||
return {
|
||||
processes: processes.items,
|
||||
...(processes.truncated ? { truncatedProcesses: processes.truncated } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function windowObservation(
|
||||
result: CuaToolResult,
|
||||
state: CuaFrameState,
|
||||
windowRef: string,
|
||||
options: { fromZoom?: boolean } = {},
|
||||
): ComputerActResult {
|
||||
const structured = projectedToolDetails(result, options.fromZoom ? "zoom" : "get_window_state");
|
||||
const observation = issueObservation(state, windowRef, options);
|
||||
const snapshotId =
|
||||
typeof structured.snapshot_id === "string" ? structured.snapshot_id : undefined;
|
||||
const rawElements = Array.isArray(structured.elements) ? structured.elements : [];
|
||||
let omittedElementCount = 0;
|
||||
const elements = rawElements.slice(0, 2_000).flatMap((entry) => {
|
||||
const parsed = NativeElementSchema.safeParse(entry);
|
||||
if (!parsed.success || !parsed.data.frame) {
|
||||
omittedElementCount += 1;
|
||||
return [];
|
||||
}
|
||||
const elementRef = issueElementRef(observation, {
|
||||
elementIndex: parsed.data.element_index,
|
||||
...(parsed.data.element_token ? { elementToken: parsed.data.element_token } : {}),
|
||||
...(snapshotId ? { snapshotId } : {}),
|
||||
});
|
||||
return [
|
||||
{
|
||||
elementRef,
|
||||
role: parsed.data.role?.trim() || "unknown",
|
||||
...(parsed.data.label !== undefined ? { label: parsed.data.label } : {}),
|
||||
...(parsed.data.value !== undefined ? { value: parsed.data.value } : {}),
|
||||
bounds: {
|
||||
x: parsed.data.frame.x,
|
||||
y: parsed.data.frame.y,
|
||||
width: parsed.data.frame.w,
|
||||
height: parsed.data.frame.h,
|
||||
},
|
||||
},
|
||||
];
|
||||
});
|
||||
const image = result.images.find((entry) => entry.mimeType === "image/png");
|
||||
const base64 = image ? canonicalizeBase64(image.dataBase64) : undefined;
|
||||
if (image && !base64) {
|
||||
throw new Error("COMPUTER_DRIVER_ERROR: CUA Driver returned malformed window PNG base64");
|
||||
}
|
||||
const width =
|
||||
typeof structured.screenshot_width === "number" && structured.screenshot_width > 0
|
||||
? Math.trunc(structured.screenshot_width)
|
||||
: undefined;
|
||||
const height =
|
||||
typeof structured.screenshot_height === "number" && structured.screenshot_height > 0
|
||||
? Math.trunc(structured.screenshot_height)
|
||||
: undefined;
|
||||
const details: Record<string, unknown> = {
|
||||
...(typeof structured.total_element_count === "number"
|
||||
? { totalElementCount: structured.total_element_count }
|
||||
: {}),
|
||||
...(rawElements.length > 2_000 ? { truncatedElements: rawElements.length - 2_000 } : {}),
|
||||
...(omittedElementCount ? { omittedElementsWithoutBounds: omittedElementCount } : {}),
|
||||
...(structured.degraded === true ? { degraded: true } : {}),
|
||||
...(typeof structured.degraded_reason === "string"
|
||||
? { degradedReason: structured.degraded_reason }
|
||||
: {}),
|
||||
...(typeof structured.screenshot_error === "string"
|
||||
? { screenshotError: structured.screenshot_error }
|
||||
: {}),
|
||||
};
|
||||
const action = actionEnvelope(result, details);
|
||||
return {
|
||||
...action,
|
||||
observation: {
|
||||
kind: "window",
|
||||
...(base64 ? { base64, format: "png" as const } : {}),
|
||||
...(width ? { width } : {}),
|
||||
...(height ? { height } : {}),
|
||||
observationId: observation.id,
|
||||
...(elements.length ? { elements } : {}),
|
||||
},
|
||||
...(!action.escalation && structured.escalation && typeof structured.escalation === "object"
|
||||
? { escalation: { recommended: "window-pixel", reasonCode: "ax_tree_unavailable" } }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
|
||||
export type CuaDesktopGeometry = {
|
||||
platform: string;
|
||||
@@ -28,12 +28,147 @@ export type CuaLastFrame = {
|
||||
export type CuaFrameState = {
|
||||
generation: string;
|
||||
lastFrame?: CuaLastFrame;
|
||||
apps?: Map<string, CuaAppTarget>;
|
||||
windows?: Map<string, CuaWindowTarget>;
|
||||
observation?: CuaObservationState;
|
||||
};
|
||||
|
||||
type CuaAppTarget = {
|
||||
pid?: number;
|
||||
name: string;
|
||||
bundleId?: string;
|
||||
launchPath?: string;
|
||||
};
|
||||
|
||||
type CuaWindowTarget = {
|
||||
pid: number;
|
||||
windowId: number;
|
||||
};
|
||||
|
||||
type CuaElementTarget = {
|
||||
elementIndex: number;
|
||||
elementToken?: string;
|
||||
snapshotId?: string;
|
||||
};
|
||||
|
||||
type CuaObservationState = {
|
||||
id: string;
|
||||
windowRef: string;
|
||||
fromZoom: boolean;
|
||||
elements: Map<string, CuaElementTarget>;
|
||||
};
|
||||
|
||||
function staleFrame(message: string): Error {
|
||||
return new Error(`COMPUTER_STALE_FRAME: ${message}; take a new screenshot`);
|
||||
}
|
||||
|
||||
function staleObservation(): Error {
|
||||
return new Error("COMPUTER_STALE_OBSERVATION: take a fresh observation and retry");
|
||||
}
|
||||
|
||||
function opaqueRef(kind: "app" | "window" | "observation" | "element"): string {
|
||||
return `cua:v2:${kind}:${randomUUID()}`;
|
||||
}
|
||||
|
||||
export function adoptGeneration(state: CuaFrameState, generation: string): void {
|
||||
// Native session replacement invalidates every authority-bearing reference,
|
||||
// even when the same window ids and display geometry reappear.
|
||||
if (state.generation !== generation) {
|
||||
state.lastFrame = undefined;
|
||||
state.apps = undefined;
|
||||
state.windows = undefined;
|
||||
state.observation = undefined;
|
||||
}
|
||||
state.generation = generation;
|
||||
}
|
||||
|
||||
export function verifyGeneration(state: CuaFrameState, generation: string): void {
|
||||
if (state.generation !== generation) {
|
||||
adoptGeneration(state, generation);
|
||||
throw staleObservation();
|
||||
}
|
||||
}
|
||||
|
||||
export function issueAppRef(state: CuaFrameState, target: CuaAppTarget): string {
|
||||
state.apps ??= new Map();
|
||||
const ref = opaqueRef("app");
|
||||
state.apps.set(ref, target);
|
||||
return ref;
|
||||
}
|
||||
|
||||
export function resolveAppRef(state: CuaFrameState, ref: string): CuaAppTarget | undefined {
|
||||
return state.apps?.get(ref);
|
||||
}
|
||||
|
||||
export function issueWindowRef(state: CuaFrameState, target: CuaWindowTarget): string {
|
||||
state.windows ??= new Map();
|
||||
for (const [ref, current] of state.windows) {
|
||||
if (current.pid === target.pid && current.windowId === target.windowId) {
|
||||
return ref;
|
||||
}
|
||||
}
|
||||
const ref = opaqueRef("window");
|
||||
state.windows.set(ref, target);
|
||||
return ref;
|
||||
}
|
||||
|
||||
export function resolveWindowRef(state: CuaFrameState, ref: string): CuaWindowTarget {
|
||||
const target = state.windows?.get(ref);
|
||||
if (!target) {
|
||||
throw staleObservation();
|
||||
}
|
||||
return target;
|
||||
}
|
||||
|
||||
export function issueObservation(
|
||||
state: CuaFrameState,
|
||||
windowRef: string,
|
||||
options: { fromZoom?: boolean } = {},
|
||||
): CuaObservationState {
|
||||
// Only the newest observation may authorize element or window-pixel actions;
|
||||
// retaining older element tokens would bypass the driver's snapshot lifecycle.
|
||||
const observation: CuaObservationState = {
|
||||
id: opaqueRef("observation"),
|
||||
windowRef,
|
||||
fromZoom: options.fromZoom === true,
|
||||
elements: new Map(),
|
||||
};
|
||||
state.observation = observation;
|
||||
return observation;
|
||||
}
|
||||
|
||||
export function issueElementRef(
|
||||
observation: CuaObservationState,
|
||||
target: CuaElementTarget,
|
||||
): string {
|
||||
const ref = opaqueRef("element");
|
||||
observation.elements.set(ref, target);
|
||||
return ref;
|
||||
}
|
||||
|
||||
export function resolveObservation(
|
||||
state: CuaFrameState,
|
||||
observationId: string,
|
||||
windowRef: string,
|
||||
): CuaObservationState {
|
||||
const observation = state.observation;
|
||||
if (!observation || observation.id !== observationId || observation.windowRef !== windowRef) {
|
||||
throw staleObservation();
|
||||
}
|
||||
return observation;
|
||||
}
|
||||
|
||||
export function resolveElementRef(
|
||||
observation: CuaObservationState,
|
||||
elementRef: string,
|
||||
): CuaElementTarget {
|
||||
const target = observation.elements.get(elementRef);
|
||||
if (!target) {
|
||||
throw staleObservation();
|
||||
}
|
||||
return target;
|
||||
}
|
||||
|
||||
/**
|
||||
* CUA Driver exposes only the primary-display label, not a stable display ID.
|
||||
* Bind authorization to connection generation plus the complete live geometry.
|
||||
|
||||
@@ -0,0 +1,524 @@
|
||||
import {
|
||||
COMPUTER_USE_V2_ACTION_NAMES,
|
||||
type ComputerActParams,
|
||||
} from "openclaw/plugin-sdk/computer-use";
|
||||
import { normalizeModifiers, parseKeyChord } from "./actions.js";
|
||||
import { EscalationReason, type CuaDriverSession } from "./driver-client.js";
|
||||
import {
|
||||
actionEnvelope,
|
||||
callWindowTool,
|
||||
nativeWindows,
|
||||
projectApps,
|
||||
projectedToolDetails,
|
||||
projectProcesses,
|
||||
projectWindows,
|
||||
windowObservation,
|
||||
} from "./driver-result.js";
|
||||
import {
|
||||
adoptGeneration,
|
||||
resolveAppRef,
|
||||
resolveElementRef,
|
||||
resolveObservation,
|
||||
resolveWindowRef,
|
||||
verifyGeneration,
|
||||
type CuaFrameState,
|
||||
} from "./frame.js";
|
||||
|
||||
const CUA_WIRE_ACTION_NAMES = COMPUTER_USE_V2_ACTION_NAMES.slice(1, 14);
|
||||
const CUA_TARGETED_ACTION_NAMES = new Set([
|
||||
"left_click",
|
||||
"right_click",
|
||||
"middle_click",
|
||||
"double_click",
|
||||
"triple_click",
|
||||
"left_click_drag",
|
||||
"left_mouse_down",
|
||||
"left_mouse_up",
|
||||
"scroll",
|
||||
"type",
|
||||
"key",
|
||||
] as const);
|
||||
|
||||
export type CuaComputerActParams = {
|
||||
action: ComputerActParams["action"];
|
||||
displayFrameId?: string;
|
||||
x?: number;
|
||||
y?: number;
|
||||
fromX?: number;
|
||||
fromY?: number;
|
||||
text?: string;
|
||||
keys?: string;
|
||||
modifiers?: string;
|
||||
scrollDirection?: "up" | "down" | "left" | "right";
|
||||
scrollAmount?: number;
|
||||
durationMs?: number;
|
||||
screenIndex?: number;
|
||||
refWidth?: number;
|
||||
windowRef?: string;
|
||||
elementRef?: string;
|
||||
observationId?: string;
|
||||
deliveryMode?: "background" | "foreground";
|
||||
query?: string;
|
||||
depth?: number;
|
||||
maxElements?: number;
|
||||
app?: string;
|
||||
value?: string;
|
||||
path?: string[];
|
||||
x1?: number;
|
||||
y1?: number;
|
||||
x2?: number;
|
||||
y2?: number;
|
||||
reason?:
|
||||
| "ax_tree_pixel_mismatch"
|
||||
| "background_delivery_failed"
|
||||
| "foreground_ineffective"
|
||||
| "no_window_target"
|
||||
| "other";
|
||||
};
|
||||
|
||||
function requireWindowTarget(
|
||||
driver: CuaDriverSession,
|
||||
state: CuaFrameState,
|
||||
params: CuaComputerActParams,
|
||||
) {
|
||||
verifyGeneration(state, driver.generation);
|
||||
if (!params.windowRef) {
|
||||
throw new Error(`COMPUTER_INVALID_REQUEST: windowRef is required for ${params.action}`);
|
||||
}
|
||||
return {
|
||||
ref: params.windowRef,
|
||||
target: resolveWindowRef(state, params.windowRef),
|
||||
};
|
||||
}
|
||||
|
||||
function observationTarget(state: CuaFrameState, params: CuaComputerActParams, windowRef: string) {
|
||||
if (!params.observationId) {
|
||||
throw new Error(`COMPUTER_STALE_OBSERVATION: observationId is required for ${params.action}`);
|
||||
}
|
||||
return resolveObservation(state, params.observationId, windowRef);
|
||||
}
|
||||
|
||||
function elementArgs(
|
||||
state: CuaFrameState,
|
||||
params: CuaComputerActParams,
|
||||
windowRef: string,
|
||||
): Record<string, unknown> | undefined {
|
||||
if (!params.elementRef) {
|
||||
return undefined;
|
||||
}
|
||||
const observation = observationTarget(state, params, windowRef);
|
||||
const element = resolveElementRef(observation, params.elementRef);
|
||||
return element.elementToken
|
||||
? { element_token: element.elementToken }
|
||||
: {
|
||||
element_index: element.elementIndex,
|
||||
...(element.snapshotId ? { snapshot_id: element.snapshotId } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function windowPointArgs(
|
||||
state: CuaFrameState,
|
||||
params: CuaComputerActParams,
|
||||
windowRef: string,
|
||||
point: { x?: number; y?: number },
|
||||
label: string,
|
||||
): Record<string, unknown> {
|
||||
if (point.x === undefined || point.y === undefined) {
|
||||
throw new Error(`COMPUTER_INVALID_REQUEST: ${label} coordinates are required`);
|
||||
}
|
||||
const observation = observationTarget(state, params, windowRef);
|
||||
return {
|
||||
x: point.x,
|
||||
y: point.y,
|
||||
...(observation.fromZoom ? { from_zoom: true } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
async function handleTargetedAct(
|
||||
platform: NodeJS.Platform,
|
||||
driver: CuaDriverSession,
|
||||
state: CuaFrameState,
|
||||
params: CuaComputerActParams,
|
||||
signal?: AbortSignal,
|
||||
): Promise<string> {
|
||||
const { ref: windowRef, target } = requireWindowTarget(driver, state, params);
|
||||
const base = { pid: target.pid, window_id: target.windowId };
|
||||
const delivery = params.deliveryMode ? { delivery_mode: params.deliveryMode } : {};
|
||||
const element = elementArgs(state, params, windowRef);
|
||||
let tool: string;
|
||||
let args: Record<string, unknown>;
|
||||
|
||||
switch (params.action) {
|
||||
case "left_click":
|
||||
case "right_click":
|
||||
case "middle_click":
|
||||
case "double_click":
|
||||
case "triple_click": {
|
||||
tool = "click";
|
||||
const button =
|
||||
params.action === "right_click"
|
||||
? "right"
|
||||
: params.action === "middle_click"
|
||||
? "middle"
|
||||
: "left";
|
||||
const count = params.action === "double_click" ? 2 : params.action === "triple_click" ? 3 : 1;
|
||||
const modifiers = normalizeModifiers(params.modifiers);
|
||||
args = {
|
||||
...base,
|
||||
...(element ?? windowPointArgs(state, params, windowRef, params, "click")),
|
||||
button,
|
||||
count,
|
||||
...(modifiers.length ? { modifier: modifiers } : {}),
|
||||
...delivery,
|
||||
};
|
||||
break;
|
||||
}
|
||||
case "left_click_drag": {
|
||||
if (element) {
|
||||
throw new Error("COMPUTER_UNSUPPORTED_ACTION: cua-driver drag has no element target");
|
||||
}
|
||||
tool = "drag";
|
||||
const from = windowPointArgs(
|
||||
state,
|
||||
params,
|
||||
windowRef,
|
||||
{ x: params.fromX, y: params.fromY },
|
||||
"drag start",
|
||||
);
|
||||
const to = windowPointArgs(state, params, windowRef, params, "drag end");
|
||||
const modifiers = normalizeModifiers(params.modifiers);
|
||||
args = {
|
||||
...base,
|
||||
from_x: from.x,
|
||||
from_y: from.y,
|
||||
to_x: to.x,
|
||||
to_y: to.y,
|
||||
...(from.from_zoom || to.from_zoom ? { from_zoom: true } : {}),
|
||||
...(params.durationMs === undefined
|
||||
? {}
|
||||
: { duration_ms: Math.min(10_000, params.durationMs) }),
|
||||
...(modifiers.length ? { modifier: modifiers } : {}),
|
||||
...delivery,
|
||||
};
|
||||
break;
|
||||
}
|
||||
case "left_mouse_down": {
|
||||
if (platform !== "linux") {
|
||||
throw new Error("COMPUTER_UNSUPPORTED_ACTION: left_mouse_down is Linux-only");
|
||||
}
|
||||
if (element || params.deliveryMode === "foreground") {
|
||||
throw new Error(
|
||||
"COMPUTER_UNSUPPORTED_ACTION: left_mouse_down supports only background window pixels",
|
||||
);
|
||||
}
|
||||
tool = "mouse_button_down";
|
||||
args = {
|
||||
...base,
|
||||
...windowPointArgs(state, params, windowRef, params, "mouse down"),
|
||||
button: "left",
|
||||
};
|
||||
break;
|
||||
}
|
||||
case "left_mouse_up": {
|
||||
if (platform !== "linux") {
|
||||
throw new Error("COMPUTER_UNSUPPORTED_ACTION: left_mouse_up is Linux-only");
|
||||
}
|
||||
if (element || params.deliveryMode === "foreground") {
|
||||
throw new Error(
|
||||
"COMPUTER_UNSUPPORTED_ACTION: left_mouse_up supports only background window pixels",
|
||||
);
|
||||
}
|
||||
tool = "mouse_button_up";
|
||||
args = {
|
||||
...base,
|
||||
...(params.x !== undefined || params.y !== undefined
|
||||
? windowPointArgs(state, params, windowRef, params, "mouse up")
|
||||
: {}),
|
||||
};
|
||||
break;
|
||||
}
|
||||
case "scroll": {
|
||||
if (!params.scrollDirection) {
|
||||
throw new Error("COMPUTER_INVALID_REQUEST: scrollDirection is required for scroll");
|
||||
}
|
||||
if (normalizeModifiers(params.modifiers).length) {
|
||||
throw new Error(
|
||||
"COMPUTER_UNSUPPORTED_ACTION: modifier-held scroll is unsupported by cua-driver",
|
||||
);
|
||||
}
|
||||
tool = "scroll";
|
||||
args = {
|
||||
...base,
|
||||
...(element ??
|
||||
(params.x !== undefined || params.y !== undefined
|
||||
? windowPointArgs(state, params, windowRef, params, "scroll")
|
||||
: {})),
|
||||
direction: params.scrollDirection,
|
||||
by: "line",
|
||||
amount: Math.min(50, params.scrollAmount ?? 3),
|
||||
...delivery,
|
||||
};
|
||||
break;
|
||||
}
|
||||
case "type": {
|
||||
if (!params.text) {
|
||||
throw new Error("COMPUTER_INVALID_REQUEST: text is required for type");
|
||||
}
|
||||
tool = "type_text";
|
||||
args = {
|
||||
...base,
|
||||
...(element ??
|
||||
(params.x !== undefined || params.y !== undefined
|
||||
? windowPointArgs(state, params, windowRef, params, "type")
|
||||
: {})),
|
||||
text: params.text,
|
||||
...delivery,
|
||||
};
|
||||
break;
|
||||
}
|
||||
case "key": {
|
||||
const chord = parseKeyChord(params.keys);
|
||||
tool = "press_key";
|
||||
args = {
|
||||
...base,
|
||||
...(element ??
|
||||
(params.x !== undefined || params.y !== undefined
|
||||
? windowPointArgs(state, params, windowRef, params, "key")
|
||||
: {})),
|
||||
key: chord.key,
|
||||
modifiers: chord.modifiers,
|
||||
...delivery,
|
||||
};
|
||||
break;
|
||||
}
|
||||
default:
|
||||
throw new Error(`COMPUTER_UNSUPPORTED_ACTION: ${params.action}`);
|
||||
}
|
||||
|
||||
const result = await callWindowTool(driver, state, tool, args, signal);
|
||||
return JSON.stringify(actionEnvelope(result));
|
||||
}
|
||||
|
||||
export async function handleV2Act(
|
||||
platform: NodeJS.Platform,
|
||||
driver: CuaDriverSession,
|
||||
state: CuaFrameState,
|
||||
params: ComputerActParams,
|
||||
handleDesktop: (
|
||||
driver: CuaDriverSession,
|
||||
state: CuaFrameState,
|
||||
params: ComputerActParams,
|
||||
signal?: AbortSignal,
|
||||
) => Promise<string>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<string> {
|
||||
const input = params as CuaComputerActParams & Record<string, unknown>;
|
||||
if (
|
||||
CUA_TARGETED_ACTION_NAMES.has(input.action as never) &&
|
||||
(input.windowRef || input.elementRef)
|
||||
) {
|
||||
return await handleTargetedAct(platform, driver, state, input, signal);
|
||||
}
|
||||
if ((CUA_WIRE_ACTION_NAMES as readonly string[]).includes(input.action)) {
|
||||
return await handleDesktop(driver, state, params, signal);
|
||||
}
|
||||
|
||||
switch (input.action) {
|
||||
case "list_apps": {
|
||||
const result = await callWindowTool(driver, state, "list_apps", {}, signal);
|
||||
state.apps = new Map();
|
||||
const structured = projectedToolDetails(result, "list_apps");
|
||||
return JSON.stringify({ ok: true, details: projectApps(state, structured.apps) });
|
||||
}
|
||||
case "list_windows": {
|
||||
const result = await callWindowTool(driver, state, "list_windows", {}, signal);
|
||||
const structured = projectedToolDetails(result, "list_windows");
|
||||
return JSON.stringify({
|
||||
ok: true,
|
||||
details: projectWindows(state, nativeWindows(structured.windows)),
|
||||
});
|
||||
}
|
||||
case "get_accessibility_tree": {
|
||||
if (input.windowRef || input.query || input.depth !== undefined || input.maxElements) {
|
||||
throw new Error(
|
||||
"COMPUTER_UNSUPPORTED_ACTION: CUA Driver 0.19.3 exposes get_accessibility_tree only as unfiltered desktop discovery; use get_window_state for a window tree",
|
||||
);
|
||||
}
|
||||
const result = await callWindowTool(driver, state, "get_accessibility_tree", {}, signal);
|
||||
const structured = projectedToolDetails(result, "get_accessibility_tree");
|
||||
return JSON.stringify({
|
||||
ok: true,
|
||||
details: {
|
||||
...projectWindows(state, nativeWindows(structured.windows)),
|
||||
...projectProcesses(structured.processes),
|
||||
},
|
||||
});
|
||||
}
|
||||
case "get_cursor_position": {
|
||||
const result = await callWindowTool(driver, state, "get_cursor_position", {}, signal);
|
||||
return JSON.stringify({
|
||||
ok: true,
|
||||
details: projectedToolDetails(result, "get_cursor_position"),
|
||||
});
|
||||
}
|
||||
case "get_window_state": {
|
||||
verifyGeneration(state, driver.generation);
|
||||
const window = resolveWindowRef(state, input.windowRef!);
|
||||
const result = await callWindowTool(
|
||||
driver,
|
||||
state,
|
||||
"get_window_state",
|
||||
{
|
||||
pid: window.pid,
|
||||
window_id: window.windowId,
|
||||
include_screenshot: true,
|
||||
max_elements: input.maxElements ?? 2_000,
|
||||
...(input.depth !== undefined ? { max_depth: Math.max(1, input.depth) } : {}),
|
||||
...(input.query ? { query: input.query } : {}),
|
||||
},
|
||||
signal,
|
||||
);
|
||||
return JSON.stringify(windowObservation(result, state, input.windowRef!));
|
||||
}
|
||||
case "launch_app": {
|
||||
verifyGeneration(state, driver.generation);
|
||||
const appName = input.app!;
|
||||
const app = resolveAppRef(state, appName);
|
||||
if (appName.startsWith("cua:v2:app:") && !app) {
|
||||
throw new Error("COMPUTER_STALE_OBSERVATION: refresh list_apps and retry");
|
||||
}
|
||||
const result = await callWindowTool(
|
||||
driver,
|
||||
state,
|
||||
"launch_app",
|
||||
app
|
||||
? app.launchPath
|
||||
? { launch_path: app.launchPath }
|
||||
: app.bundleId
|
||||
? { bundle_id: app.bundleId }
|
||||
: { name: app.name }
|
||||
: { name: appName },
|
||||
signal,
|
||||
);
|
||||
const structured = projectedToolDetails(result, "launch_app");
|
||||
return JSON.stringify({
|
||||
...actionEnvelope(result),
|
||||
details: {
|
||||
app: projectApps(state, [structured]).apps,
|
||||
...projectWindows(state, nativeWindows(structured.windows)),
|
||||
},
|
||||
});
|
||||
}
|
||||
case "kill_app": {
|
||||
verifyGeneration(state, driver.generation);
|
||||
const appName = input.app!;
|
||||
const app = resolveAppRef(state, appName);
|
||||
if (!app?.pid) {
|
||||
throw new Error(
|
||||
"COMPUTER_INVALID_REQUEST: kill_app requires a running app reference from list_apps",
|
||||
);
|
||||
}
|
||||
const result = await callWindowTool(driver, state, "kill_app", { pid: app.pid }, signal);
|
||||
return JSON.stringify(actionEnvelope(result, { app: appName }));
|
||||
}
|
||||
case "bring_to_front": {
|
||||
const { target } = requireWindowTarget(driver, state, input);
|
||||
const result = await callWindowTool(
|
||||
driver,
|
||||
state,
|
||||
"bring_to_front",
|
||||
{
|
||||
pid: target.pid,
|
||||
window_id: target.windowId,
|
||||
},
|
||||
signal,
|
||||
);
|
||||
return JSON.stringify(actionEnvelope(result));
|
||||
}
|
||||
case "set_value": {
|
||||
const { ref, target } = requireWindowTarget(driver, state, input);
|
||||
if (input.deliveryMode === "foreground") {
|
||||
throw new Error(
|
||||
"COMPUTER_UNSUPPORTED_ACTION: cua-driver set_value is background accessibility delivery",
|
||||
);
|
||||
}
|
||||
const element = elementArgs(state, input, ref);
|
||||
if (!element) {
|
||||
throw new Error("COMPUTER_INVALID_REQUEST: elementRef is required for set_value");
|
||||
}
|
||||
const result = await callWindowTool(
|
||||
driver,
|
||||
state,
|
||||
"set_value",
|
||||
{
|
||||
pid: target.pid,
|
||||
window_id: target.windowId,
|
||||
...element,
|
||||
value: input.value,
|
||||
},
|
||||
signal,
|
||||
);
|
||||
return JSON.stringify(actionEnvelope(result));
|
||||
}
|
||||
case "invoke_menu": {
|
||||
const { target } = requireWindowTarget(driver, state, input);
|
||||
if (input.deliveryMode === "foreground") {
|
||||
throw new Error(
|
||||
"COMPUTER_UNSUPPORTED_ACTION: cua-driver invoke_menu is background accessibility delivery",
|
||||
);
|
||||
}
|
||||
const result = await callWindowTool(
|
||||
driver,
|
||||
state,
|
||||
"invoke_menu",
|
||||
{
|
||||
pid: target.pid,
|
||||
window_id: target.windowId,
|
||||
path: input.path,
|
||||
},
|
||||
signal,
|
||||
);
|
||||
return JSON.stringify(actionEnvelope(result));
|
||||
}
|
||||
case "zoom": {
|
||||
const { ref, target } = requireWindowTarget(driver, state, input);
|
||||
resolveObservation(state, input.observationId!, ref);
|
||||
const result = await callWindowTool(
|
||||
driver,
|
||||
state,
|
||||
"zoom",
|
||||
{
|
||||
pid: target.pid,
|
||||
window_id: target.windowId,
|
||||
x1: input.x1,
|
||||
y1: input.y1,
|
||||
x2: input.x2,
|
||||
y2: input.y2,
|
||||
},
|
||||
signal,
|
||||
);
|
||||
return JSON.stringify(windowObservation(result, state, ref, { fromZoom: true }));
|
||||
}
|
||||
case "escalate_scope": {
|
||||
const reason = {
|
||||
ax_tree_pixel_mismatch: EscalationReason.AxTreePixelMismatch,
|
||||
background_delivery_failed: EscalationReason.BackgroundDeliveryFailed,
|
||||
foreground_ineffective: EscalationReason.ForegroundIneffective,
|
||||
no_window_target: EscalationReason.NoWindowTarget,
|
||||
other: EscalationReason.Other,
|
||||
}[input.reason!];
|
||||
const result = await driver.escalateScope(reason, signal);
|
||||
adoptGeneration(state, driver.generation);
|
||||
return JSON.stringify({
|
||||
ok: true,
|
||||
details: {
|
||||
captureScope: result.captureScope,
|
||||
effectiveScope: result.effectiveScope,
|
||||
desktopUnlocked: result.desktopUnlocked,
|
||||
},
|
||||
});
|
||||
}
|
||||
default:
|
||||
throw new Error(`COMPUTER_UNSUPPORTED_ACTION: ${input.action}`);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user