diff --git a/extensions/cua-computer/src/commands.test.ts b/extensions/cua-computer/src/commands.test.ts index 248fe191341a..d404ba34e8cd 100644 --- a/extensions/cua-computer/src/commands.test.ts +++ b/extensions/cua-computer/src/commands.test.ts @@ -1,7 +1,12 @@ import { describe, expect, it, vi } from "vitest"; import { createCuaComputerProvider } from "./commands.js"; +import { + CUA_DRIVER_CONTRACT_FIXTURES, + cuaToolResult, +} from "./cua-driver-contract.test-fixtures.js"; import { ClickButton, + EscalationReason, ScrollDirection, type CuaDriverSession, type CuaToolResult, @@ -31,6 +36,7 @@ function result(structured: Record, image = false): CuaToolResu } function driver() { + let generation = "execution-1"; const getDesktopState = vi.fn(async () => result(geometry, true)); const getScreenSize = vi.fn(async () => result({ width: 100, height: 50, scale_factor: 1 })); const click = vi.fn(async () => result({})); @@ -39,11 +45,22 @@ function driver() { const scroll = vi.fn(async () => result({})); const typeText = vi.fn(async () => result({})); const pressKey = vi.fn(async () => result({})); + const callTool = vi.fn(async () => result({})); + const escalateScope = vi.fn(async () => ({ + session: "openclaw-test", + captureScope: 2, + effectiveScope: 1, + desktopUnlocked: true, + })); const dispose = vi.fn(async () => {}); const session: CuaDriverSession = { - generation: "execution-1", + get generation() { + return generation; + }, isAvailable: () => true, resetAvailabilityCache: () => {}, + callTool, + escalateScope, getDesktopState, getScreenSize, click, @@ -62,9 +79,14 @@ function driver() { drag, moveCursor, scroll, + callTool, + escalateScope, dispose, typeText, pressKey, + setGeneration: (value: string) => { + generation = value; + }, }; } @@ -79,7 +101,7 @@ async function execution(session: CuaDriverSession) { } describe("cua-computer provider", () => { - it("advertises only its current foreground coordinate capability", () => { + it("advertises the implemented Linux v2 capability", () => { const { session } = driver(); const descriptor = createCuaComputerProvider({ platform: "linux", @@ -90,7 +112,7 @@ describe("cua-computer provider", () => { provider: { id: "cua-computer", label: "CUA Computer", - generation: "cua-computer-coordinate-v1", + generation: "cua-computer-v2:execution-1", }, actions: [ "screenshot", @@ -106,16 +128,35 @@ describe("cua-computer provider", () => { "scroll", "type", "key", - "hold_key", - "wait", + "list_apps", + "list_windows", + "get_accessibility_tree", + "get_cursor_position", + "get_window_state", + "launch_app", + "kill_app", + "bring_to_front", + "set_value", + "zoom", + "escalate_scope", + "invoke_menu", ], - targets: ["screen"], - deliveryModes: ["foreground"], - observations: ["image"], + targets: ["screen", "window", "element"], + deliveryModes: ["background", "foreground"], + observations: ["image", "accessibility"], features: { recording: false, agentCursor: false, multiDisplay: false }, }); }); + it("omits Linux-only held-button actions on Windows", () => { + const { session } = driver(); + const actions = createCuaComputerProvider({ platform: "win32", driver: session }).capabilities() + .actions; + expect(actions).not.toContain("left_mouse_down"); + expect(actions).not.toContain("left_mouse_up"); + expect(actions).toContain("get_window_state"); + }); + it("uses one typed session for snapshot and frame-authorized click", async () => { const { session, getDesktopState, getScreenSize, click } = driver(); const computer = await execution(session); @@ -279,4 +320,359 @@ describe("cua-computer provider", () => { await computer.snapshot('{"format":"png","maxWidth":100}', signal); expect(getDesktopState).toHaveBeenCalledWith(signal); }); + + it("mints opaque window and element references and maps background evidence", async () => { + const { session, callTool } = driver(); + callTool.mockImplementation(async (name) => { + switch (name) { + case "list_windows": + return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.listWindows); + case "get_window_state": + return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.windowState, { image: true }); + case "click": + return cuaToolResult( + {}, + { + action: + CUA_DRIVER_CONTRACT_FIXTURES.confirmedBackgroundAction as unknown as CuaToolResult["action"], + }, + ); + default: + return cuaToolResult({}); + } + }); + const computer = await execution(session); + const listed = JSON.parse(await computer.act('{"action":"list_windows"}')) as { + details: { windows: Array<{ windowRef: string }> }; + }; + const windowRef = listed.details.windows[0]!.windowRef; + expect(windowRef).toMatch(/^cua:v2:window:/); + + const observed = JSON.parse( + await computer.act(JSON.stringify({ action: "get_window_state", windowRef })), + ) as { + observation: { + observationId: string; + elements: Array<{ elementRef: string }>; + }; + }; + const { observationId } = observed.observation; + const elementRef = observed.observation.elements[0]!.elementRef; + expect(observationId).toMatch(/^cua:v2:observation:/); + expect(elementRef).toMatch(/^cua:v2:element:/); + + const clicked = JSON.parse( + await computer.act( + JSON.stringify({ + action: "left_click", + windowRef, + elementRef, + observationId, + deliveryMode: "background", + }), + ), + ) as { effect: string; details: Record }; + expect(clicked).toMatchObject({ + ok: true, + effect: "confirmed", + details: { + route: "accessibility", + deliveryMode: "background", + deliveredCount: 1, + evidence: ["value_readback"], + }, + }); + expect(callTool).toHaveBeenLastCalledWith( + "click", + { + pid: 4242, + window_id: 99, + element_token: "native-element-token-7", + button: "left", + count: 1, + delivery_mode: "background", + }, + undefined, + ); + }); + + it("maps window pixels, app lifecycle, menu, zoom, and escalation tools", async () => { + const { session, callTool, escalateScope } = driver(); + callTool.mockImplementation(async (name) => { + switch (name) { + case "list_apps": + return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.listApps); + case "list_windows": + return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.listWindows); + case "get_window_state": + return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.windowState, { image: true }); + case "zoom": + return cuaToolResult({ screenshot_width: 300, screenshot_height: 200 }, { image: true }); + default: + return cuaToolResult( + {}, + { + action: + CUA_DRIVER_CONTRACT_FIXTURES.suspectedNoopAction as unknown as CuaToolResult["action"], + }, + ); + } + }); + const computer = await execution(session); + const apps = JSON.parse(await computer.act('{"action":"list_apps"}')) as { + details: { apps: Array<{ app: string }> }; + }; + const app = apps.details.apps[0]!.app; + const windows = JSON.parse(await computer.act('{"action":"list_windows"}')) as { + details: { windows: Array<{ windowRef: string }> }; + }; + const windowRef = windows.details.windows[0]!.windowRef; + const observed = JSON.parse( + await computer.act(JSON.stringify({ action: "get_window_state", windowRef })), + ) as { observation: { observationId: string } }; + + await computer.act(JSON.stringify({ action: "launch_app", app })); + await computer.act(JSON.stringify({ action: "kill_app", app })); + await computer.act( + JSON.stringify({ action: "invoke_menu", windowRef, path: ["File", "Save"] }), + ); + const zoomed = JSON.parse( + await computer.act( + JSON.stringify({ + action: "zoom", + windowRef, + observationId: observed.observation.observationId, + x1: 0, + y1: 0, + x2: 100, + y2: 100, + }), + ), + ) as { observation: { observationId: string } }; + expect(zoomed.observation.observationId).not.toBe(observed.observation.observationId); + await computer.act( + JSON.stringify({ action: "escalate_scope", reason: "background_delivery_failed" }), + ); + + expect(callTool).toHaveBeenCalledWith( + "launch_app", + { launch_path: "/usr/bin/editor" }, + undefined, + ); + expect(callTool).toHaveBeenCalledWith("kill_app", { pid: 4242 }, undefined); + expect(callTool).toHaveBeenCalledWith( + "invoke_menu", + { pid: 4242, window_id: 99, path: ["File", "Save"] }, + undefined, + ); + expect(escalateScope).toHaveBeenCalledWith( + EscalationReason.BackgroundDeliveryFailed, + undefined, + ); + }); + + it("maps the complete Linux window pointer and keyboard family", async () => { + const { session, callTool } = driver(); + callTool.mockImplementation(async (name) => { + if (name === "list_windows") { + return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.listWindows); + } + if (name === "get_window_state") { + return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.windowState, { image: true }); + } + return cuaToolResult( + {}, + { + action: + CUA_DRIVER_CONTRACT_FIXTURES.confirmedBackgroundAction as unknown as CuaToolResult["action"], + }, + ); + }); + const computer = await execution(session); + const listed = JSON.parse(await computer.act('{"action":"list_windows"}')) as { + details: { windows: Array<{ windowRef: string }> }; + }; + const windowRef = listed.details.windows[0]!.windowRef; + const observed = JSON.parse( + await computer.act(JSON.stringify({ action: "get_window_state", windowRef })), + ) as { + observation: { observationId: string; elements: Array<{ elementRef: string }> }; + }; + const observationId = observed.observation.observationId; + const elementRef = observed.observation.elements[0]!.elementRef; + const pixelTarget = { windowRef, observationId, x: 20, y: 30 }; + const cases = [ + ["right_click", "click", { button: "right", count: 1 }], + ["middle_click", "click", { button: "middle", count: 1 }], + ["double_click", "click", { button: "left", count: 2 }], + ["triple_click", "click", { button: "left", count: 3 }], + ["left_click_drag", "drag", { from_x: 10, from_y: 15, to_x: 20, to_y: 30, duration_ms: 250 }], + ["left_mouse_down", "mouse_button_down", { x: 20, y: 30, button: "left" }], + ["left_mouse_up", "mouse_button_up", { x: 20, y: 30 }], + ["scroll", "scroll", { direction: "down", by: "line", amount: 4 }], + ["type", "type_text", { text: "hello", element_token: "native-element-token-7" }], + ["key", "press_key", { key: "enter", modifiers: ["ctrl"] }], + ] as const; + + for (const [action, tool, expected] of cases) { + const actionInput: Record = { + action, + ...pixelTarget, + deliveryMode: action.startsWith("left_mouse_") ? "background" : "foreground", + }; + if (action === "left_click_drag") { + actionInput.fromX = 10; + actionInput.fromY = 15; + actionInput.durationMs = 250; + } else if (action === "scroll") { + actionInput.scrollDirection = "down"; + actionInput.scrollAmount = 4; + } else if (action === "type") { + actionInput.elementRef = elementRef; + actionInput.text = "hello"; + delete actionInput.x; + delete actionInput.y; + } else if (action === "key") { + actionInput.keys = "ctrl+enter"; + delete actionInput.x; + delete actionInput.y; + } + await computer.act(JSON.stringify(actionInput)); + expect(callTool).toHaveBeenCalledWith( + tool, + expect.objectContaining({ pid: 4242, window_id: 99, ...expected }), + undefined, + ); + } + }); + + it("maps remaining discovery, window lifecycle, and semantic actions", async () => { + const { session, callTool } = driver(); + callTool.mockImplementation(async (name) => { + if (name === "list_windows") { + return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.listWindows); + } + if (name === "get_window_state") { + return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.windowState, { image: true }); + } + if (name === "get_accessibility_tree") { + return cuaToolResult({ + processes: [{ pid: 4242, name: "Editor" }], + windows: CUA_DRIVER_CONTRACT_FIXTURES.listWindows.windows, + }); + } + if (name === "get_cursor_position") { + return cuaToolResult({ x: 11, y: 12, source: "x11" }); + } + return cuaToolResult( + {}, + { + action: + CUA_DRIVER_CONTRACT_FIXTURES.confirmedBackgroundAction as unknown as CuaToolResult["action"], + }, + ); + }); + const computer = await execution(session); + const tree = JSON.parse(await computer.act('{"action":"get_accessibility_tree"}')) as { + details: { windows: unknown[]; processes: unknown[] }; + }; + expect(tree.details.windows).toHaveLength(1); + expect(tree.details.processes).toHaveLength(1); + await expect(computer.act('{"action":"get_cursor_position"}')).resolves.toContain('"x":11'); + + const listed = JSON.parse(await computer.act('{"action":"list_windows"}')) as { + details: { windows: Array<{ windowRef: string }> }; + }; + const windowRef = listed.details.windows[0]!.windowRef; + const observed = JSON.parse( + await computer.act(JSON.stringify({ action: "get_window_state", windowRef })), + ) as { + observation: { observationId: string; elements: Array<{ elementRef: string }> }; + }; + await computer.act(JSON.stringify({ action: "bring_to_front", windowRef })); + await computer.act( + JSON.stringify({ + action: "set_value", + windowRef, + observationId: observed.observation.observationId, + elementRef: observed.observation.elements[0]!.elementRef, + value: "new", + deliveryMode: "background", + }), + ); + expect(callTool).toHaveBeenCalledWith( + "bring_to_front", + { pid: 4242, window_id: 99 }, + undefined, + ); + expect(callTool).toHaveBeenCalledWith( + "set_value", + { + pid: 4242, + window_id: 99, + element_token: "native-element-token-7", + value: "new", + }, + undefined, + ); + }); + + it("maps window delivery refusals to the closed computer error prefix", async () => { + const { session, callTool } = driver(); + callTool.mockImplementation(async (name) => { + if (name === "list_windows") { + return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.listWindows); + } + if (name === "get_window_state") { + return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.windowState, { image: true }); + } + return cuaToolResult( + { code: "background_occluded" }, + { + isError: true, + errorCode: "background_occluded", + text: "target is occluded", + }, + ); + }); + const computer = await execution(session); + const listed = JSON.parse(await computer.act('{"action":"list_windows"}')) as { + details: { windows: Array<{ windowRef: string }> }; + }; + const windowRef = listed.details.windows[0]!.windowRef; + const observed = JSON.parse( + await computer.act(JSON.stringify({ action: "get_window_state", windowRef })), + ) as { observation: { observationId: string } }; + await expect( + computer.act( + JSON.stringify({ + action: "left_click", + windowRef, + observationId: observed.observation.observationId, + x: 10, + y: 20, + }), + ), + ).rejects.toThrow("COMPUTER_REFUSED_background_occluded"); + }); + + it("invalidates observation references when the driver generation rotates", async () => { + const { session, callTool, setGeneration } = driver(); + callTool.mockImplementation(async (name) => + name === "list_windows" + ? cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.listWindows) + : cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.windowState, { image: true }), + ); + const computer = await execution(session); + const listed = JSON.parse(await computer.act('{"action":"list_windows"}')) as { + details: { windows: Array<{ windowRef: string }> }; + }; + const windowRef = listed.details.windows[0]!.windowRef; + setGeneration("execution-2"); + + await expect( + computer.act(JSON.stringify({ action: "get_window_state", windowRef })), + ).rejects.toThrow("COMPUTER_STALE_OBSERVATION"); + expect(callTool).toHaveBeenCalledTimes(1); + }); }); diff --git a/extensions/cua-computer/src/commands.ts b/extensions/cua-computer/src/commands.ts index 8e007d0cc439..4e060d17b740 100644 --- a/extensions/cua-computer/src/commands.ts +++ b/extensions/cua-computer/src/commands.ts @@ -19,7 +19,9 @@ import { type CuaDriverSession, type CuaToolResult, } from "./driver-client.js"; +import { platformActions } from "./driver-result.js"; import { + adoptGeneration, issueFrame, verifyFrame, verifyReferenceWidth, @@ -28,26 +30,10 @@ import { type CuaLastFrame, type CuaScreenSize, } from "./frame.js"; +import { handleV2Act, type CuaComputerActParams } from "./v2-actions.js"; const AVAILABILITY_POLL_MS = 5_000; -const CUA_COORDINATE_ACTION_NAMES = COMPUTER_USE_V2_ACTION_NAMES.slice(0, 15); const CUA_WIRE_ACTION_NAMES = COMPUTER_USE_V2_ACTION_NAMES.slice(1, 14); -type CuaComputerActParams = { - action: ComputerActParams["action"]; - displayFrameId?: string; - x?: number; - y?: number; - fromX?: number; - fromY?: number; - text?: string; - keys?: string; - modifiers?: string; - scrollDirection?: "up" | "down" | "left" | "right"; - scrollAmount?: number; - durationMs?: number; - screenIndex?: number; - refWidth?: number; -}; // Rastermill enforces inputPixels before resizing, so this must clear the native // capture, not the delivered frame. 8K (7680x4320 = ~33.2M) is a valid primary // display; budget above it so full-resolution snapshots reach the downscaler. @@ -244,7 +230,7 @@ async function currentFrame( return frame; } -async function handleAct( +async function handleDesktopAct( driver: CuaDriverSession, frameState: CuaFrameState, params: ComputerActParams, @@ -441,12 +427,14 @@ export function createCuaComputerProvider( provider: { id: "cua-computer", label: "CUA Computer", - generation: "cua-computer-coordinate-v1", + generation: isSupportedPlatform + ? `cua-computer-v2:${driver().generation}` + : "cua-computer-v2:unsupported", }, - actions: CUA_COORDINATE_ACTION_NAMES, - targets: ["screen"], - deliveryModes: ["foreground"], - observations: ["image"], + actions: platformActions(platform), + targets: ["screen", "window", "element"], + deliveryModes: ["background", "foreground"], + observations: ["image", "accessibility"], features: { recording: false, agentCursor: false, multiDisplay: false }, }), isAvailable, @@ -468,7 +456,7 @@ export function createCuaComputerProvider( }, openExecution: async () => { const queue = new PromiseQueue(); - const frameState: CuaFrameState = { generation: "uninitialized" }; + const frameState: CuaFrameState = { generation: driver().generation }; return { snapshot: async (paramsJSON, signal) => await queue.run(async () => { @@ -511,7 +499,7 @@ export function createCuaComputerProvider( width = result.width; height = result.height; } - frameState.generation = driver().generation; + adoptGeneration(frameState, driver().generation); const displayFrameId = issueFrame(frameState, geometry, { width, height }); return JSON.stringify({ format, @@ -529,10 +517,12 @@ export function createCuaComputerProvider( "COMPUTER_DRIVER_UNAVAILABLE: cua-computer supports Windows and Linux", ); } - return await handleAct( + return await handleV2Act( + platform, driver(), frameState, parseComputerActParamsJSON(paramsJSON), + handleDesktopAct, signal, ); }), diff --git a/extensions/cua-computer/src/cua-driver-contract.test-fixtures.ts b/extensions/cua-computer/src/cua-driver-contract.test-fixtures.ts new file mode 100644 index 000000000000..9fe91e08b15a --- /dev/null +++ b/extensions/cua-computer/src/cua-driver-contract.test-fixtures.ts @@ -0,0 +1,88 @@ +import type { CuaToolResult } from "./driver-client.js"; + +export const CUA_DRIVER_CONTRACT_FIXTURES = { + listApps: { + apps: [ + { + pid: 4242, + bundle_id: "org.example.Editor", + name: "Editor", + running: true, + active: false, + kind: "desktop", + launch_path: "/usr/bin/editor", + last_used: "2026-08-14T00:00:00Z", + }, + ], + }, + listWindows: { + windows: [ + { + window_id: 99, + pid: 4242, + app_name: "Editor", + title: "Notes", + bounds: { x: 40, y: 50, width: 800, height: 600 }, + is_on_screen: true, + minimized: false, + z_index: 2, + }, + ], + }, + windowState: { + window_id: 99, + pid: 4242, + snapshot_id: "native-snapshot-1", + total_element_count: 1, + returned_element_count: 1, + screenshot_width: 800, + screenshot_height: 600, + screenshot_mime_type: "image/png", + elements: [ + { + element_index: 7, + element_token: "native-element-token-7", + role: "text field", + label: "Body", + value: "old", + frame: { x: 80, y: 100, w: 400, h: 240 }, + }, + ], + }, + confirmedBackgroundAction: { + effect: 0, + route: 0, + delivery: { mode: 0, deliveredCount: 1 }, + evidence: [{ kind: 0 }], + }, + suspectedNoopAction: { + effect: 3, + route: 1, + delivery: { mode: 0 }, + escalation: { target: 1, reason: 3 }, + }, +} as const; + +export function cuaToolResult( + structured: Record, + options: { + action?: CuaToolResult["action"]; + image?: boolean; + isError?: boolean; + errorCode?: string; + text?: string; + } = {}, +): CuaToolResult { + return { + text: options.text ?? "ok", + images: options.image + ? [{ mimeType: "image/png", dataBase64: Buffer.from("png").toString("base64") }] + : [], + structuredJson: JSON.stringify(structured), + isError: options.isError ?? false, + ...(options.errorCode ? { errorCode: options.errorCode } : {}), + ...(options.action ? { action: options.action } : {}), + degraded: false, + rawJson: "{}", + }; +} diff --git a/extensions/cua-computer/src/driver-client.test.ts b/extensions/cua-computer/src/driver-client.test.ts index d6f44d091e4f..fa74c95a8352 100644 --- a/extensions/cua-computer/src/driver-client.test.ts +++ b/extensions/cua-computer/src/driver-client.test.ts @@ -2,10 +2,17 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; const mocks = vi.hoisted(() => ({ close: vi.fn(), + callTool: vi.fn(async () => ({})), create: vi.fn(), createConfigured: vi.fn(), createTrustedSession: vi.fn(), endSession: vi.fn(async () => ({})), + escalateSession: vi.fn(async () => ({ + session: "openclaw-test", + captureScope: "desktop", + effectiveScope: "desktop", + desktopUnlocked: true, + })), getDesktopState: vi.fn(async () => ({})), isAvailable: vi.fn(() => true), startSession: vi.fn(async () => ({})), @@ -13,10 +20,11 @@ const mocks = vi.hoisted(() => ({ })); const sdk = { - CaptureScope: { Desktop: "desktop" }, + CaptureScope: { Window: "window", Desktop: "desktop" }, ClickButton: { Left: 0, Right: 1, Middle: 2 }, CuaDriver: { create: mocks.create, createConfigured: mocks.createConfigured }, DesktopScope: { Desktop: 0 }, + EscalationReason: { Other: "other" }, ScrollBy: { Line: 0 }, ScrollDirection: { Up: 0, Down: 1, Left: 2, Right: 3 }, SessionPermissionMode: { Unrestricted: "unrestricted" }, @@ -42,7 +50,9 @@ describe("CUA Driver direct session", () => { }); mocks.createTrustedSession.mockReturnValue({ close: mocks.close, + callTool: mocks.callTool, endSession: mocks.endSession, + escalateSession: mocks.escalateSession, getDesktopState: mocks.getDesktopState, startSession: mocks.startSession, }); @@ -109,6 +119,33 @@ describe("CUA Driver direct session", () => { expect(mocks.endSession).toHaveBeenCalledWith({ session: sessionOptions.publicSession }); }); + it("starts window-scoped generic tools and widens only for an explicit desktop call", async () => { + const driver = createCuaDriver({ loadSdk: () => sdk as never }); + + await driver.callTool("list_windows", {}); + const sessionOptions = mocks.createTrustedSession.mock.calls[0]?.[1]; + expect(mocks.startSession).toHaveBeenCalledWith( + { session: sessionOptions.publicSession, captureScope: "window" }, + undefined, + ); + expect(mocks.callTool).toHaveBeenCalledWith( + "list_windows", + JSON.stringify({ session: sessionOptions.publicSession }), + undefined, + ); + + await driver.getDesktopState(); + expect(mocks.escalateSession).toHaveBeenCalledWith( + { + session: sessionOptions.publicSession, + reason: "other", + detail: "explicit desktop-scope OpenClaw action", + }, + undefined, + ); + await driver.dispose(); + }); + it("keeps a missing native desktop library behind command availability", async () => { const loadSdk = vi.fn(() => { throw new Error("libX11.so.6: cannot open shared object file"); diff --git a/extensions/cua-computer/src/driver-client.ts b/extensions/cua-computer/src/driver-client.ts index 832f746c2d1c..5112e51cbaf7 100644 --- a/extensions/cua-computer/src/driver-client.ts +++ b/extensions/cua-computer/src/driver-client.ts @@ -1,14 +1,18 @@ import { randomUUID } from "node:crypto"; type DriverClickButton = import("@trycua/cua-driver").ClickButton; +type DriverCaptureScope = import("@trycua/cua-driver").CaptureScope; +type DriverEscalationReason = import("@trycua/cua-driver").EscalationReason; type CuaDriverLike = import("@trycua/cua-driver").CuaDriverLike; type CuaDriverSessionLike = import("@trycua/cua-driver").CuaDriverSessionLike; type DriverScrollDirection = import("@trycua/cua-driver").ScrollDirection; +type CuaSessionState = import("@trycua/cua-driver").SessionStateOutput; type CuaDriverSdk = Pick< typeof import("@trycua/cua-driver"), | "CaptureScope" | "CuaDriver" | "DesktopScope" + | "EscalationReason" | "ScrollBy" | "SessionPermissionMode" | "createTrustedSession" @@ -16,6 +20,15 @@ type CuaDriverSdk = Pick< export type CuaToolResult = import("@trycua/cua-driver").ToolResult; +export const EscalationReason = { + AxTreePixelMismatch: 0 as DriverEscalationReason, + BackgroundDeliveryFailed: 1 as DriverEscalationReason, + ForegroundIneffective: 2 as DriverEscalationReason, + NoWindowTarget: 3 as DriverEscalationReason, + Other: 4 as DriverEscalationReason, +} as const; +export type EscalationReason = (typeof EscalationReason)[keyof typeof EscalationReason]; + // These numeric values are part of the pinned 0.19.3 SDK contract. Keeping // them local avoids loading the native library while OpenClaw is only // registering the bundled plugin. @@ -38,6 +51,12 @@ export interface CuaDriverSession { readonly generation: string; isAvailable(): boolean; resetAvailabilityCache(): void; + callTool( + name: string, + args: Record, + signal?: AbortSignal, + ): Promise; + escalateScope(reason: EscalationReason, signal?: AbortSignal): Promise; getDesktopState(signal?: AbortSignal): Promise; getScreenSize(signal?: AbortSignal): Promise; click( @@ -71,6 +90,8 @@ class DirectCuaDriverSession implements CuaDriverSession { private readonly session: CuaDriverSessionLike; private readonly publicSession = `openclaw-${randomUUID()}`; private startPromise: Promise | undefined; + private desktopEscalationPromise: Promise | undefined; + private captureScope: DriverCaptureScope | undefined; private started = false; private disposed = false; @@ -99,16 +120,17 @@ class DirectCuaDriverSession implements CuaDriverSession { }); } - private async ensureStarted(signal?: AbortSignal): Promise { + private async ensureStarted( + captureScope: DriverCaptureScope, + signal?: AbortSignal, + ): Promise { if (this.disposed) { throw new Error("COMPUTER_DRIVER_UNAVAILABLE: cua-computer is stopping"); } if (!this.startPromise) { + this.captureScope = captureScope; const start = this.session - .startSession( - { session: this.publicSession, captureScope: this.sdk.CaptureScope.Desktop }, - asyncOptions(signal), - ) + .startSession({ session: this.publicSession, captureScope }, asyncOptions(signal)) .then(() => { this.started = true; }); @@ -124,13 +146,38 @@ class DirectCuaDriverSession implements CuaDriverSession { return; } await this.startPromise; + if ( + captureScope === this.sdk.CaptureScope.Desktop && + this.captureScope !== this.sdk.CaptureScope.Desktop + ) { + await this.ensureDesktopScope(signal); + } + } + + private async ensureDesktopScope(signal?: AbortSignal): Promise { + if (!this.desktopEscalationPromise) { + this.desktopEscalationPromise = this.session + .escalateSession( + { + session: this.publicSession, + reason: this.sdk.EscalationReason.Other, + detail: "explicit desktop-scope OpenClaw action", + }, + asyncOptions(signal), + ) + .then(() => { + this.captureScope = this.sdk.CaptureScope.Desktop; + }); + } + await this.desktopEscalationPromise; } private async invoke( + captureScope: DriverCaptureScope, signal: AbortSignal | undefined, operation: () => Promise, ): Promise { - await this.ensureStarted(signal); + await this.ensureStarted(captureScope, signal); return await operation(); } @@ -138,17 +185,39 @@ class DirectCuaDriverSession implements CuaDriverSession { return !this.disposed && this.runtime.isAvailable(); } resetAvailabilityCache(): void {} + async callTool(name: string, args: Record, signal?: AbortSignal) { + return await this.invoke(this.sdk.CaptureScope.Window, signal, () => + this.session.callTool( + name, + JSON.stringify({ ...args, session: this.publicSession }), + asyncOptions(signal), + ), + ); + } + async escalateScope(reason: EscalationReason, signal?: AbortSignal) { + await this.ensureStarted(this.sdk.CaptureScope.Window, signal); + const state = await this.session.escalateSession( + { session: this.publicSession, reason }, + asyncOptions(signal), + ); + this.captureScope = this.sdk.CaptureScope.Desktop; + return state; + } async getDesktopState(signal?: AbortSignal) { - return await this.invoke(signal, () => this.session.getDesktopState({}, asyncOptions(signal))); + return await this.invoke(this.sdk.CaptureScope.Desktop, signal, () => + this.session.getDesktopState({}, asyncOptions(signal)), + ); } async getScreenSize(signal?: AbortSignal) { - return await this.invoke(signal, () => this.session.getScreenSize({}, asyncOptions(signal))); + return await this.invoke(this.sdk.CaptureScope.Desktop, signal, () => + this.session.getScreenSize({}, asyncOptions(signal)), + ); } async click( input: { x: number; y: number; button: ClickButton; count: number }, signal?: AbortSignal, ) { - return await this.invoke(signal, () => + return await this.invoke(this.sdk.CaptureScope.Desktop, signal, () => this.session.click({ ...input, scope: this.sdk.DesktopScope.Desktop }, asyncOptions(signal)), ); } @@ -156,12 +225,12 @@ class DirectCuaDriverSession implements CuaDriverSession { input: { fromX: number; fromY: number; toX: number; toY: number; durationMs?: bigint }, signal?: AbortSignal, ) { - return await this.invoke(signal, () => + return await this.invoke(this.sdk.CaptureScope.Desktop, signal, () => this.session.drag({ ...input, scope: this.sdk.DesktopScope.Desktop }, asyncOptions(signal)), ); } async moveCursor(input: { x: number; y: number }, signal?: AbortSignal) { - return await this.invoke(signal, () => + return await this.invoke(this.sdk.CaptureScope.Desktop, signal, () => this.session.moveCursor( { ...input, scope: this.sdk.DesktopScope.Desktop }, asyncOptions(signal), @@ -172,7 +241,7 @@ class DirectCuaDriverSession implements CuaDriverSession { input: { x: number; y: number; direction: ScrollDirection; amount: bigint }, signal?: AbortSignal, ) { - return await this.invoke(signal, () => + return await this.invoke(this.sdk.CaptureScope.Desktop, signal, () => this.session.scroll( { ...input, @@ -184,12 +253,12 @@ class DirectCuaDriverSession implements CuaDriverSession { ); } async typeText(text: string, signal?: AbortSignal) { - return await this.invoke(signal, () => + return await this.invoke(this.sdk.CaptureScope.Desktop, signal, () => this.session.typeText({ text, scope: this.sdk.DesktopScope.Desktop }, asyncOptions(signal)), ); } async pressKey(input: { key: string; modifiers: string[] }, signal?: AbortSignal) { - return await this.invoke(signal, () => + return await this.invoke(this.sdk.CaptureScope.Desktop, signal, () => this.session.pressKey( { ...input, scope: this.sdk.DesktopScope.Desktop }, asyncOptions(signal), @@ -344,6 +413,12 @@ class LazyCuaDriverSession implements CuaDriverSession { async getDesktopState(signal?: AbortSignal) { return await (await this.requireRuntime()).getDesktopState(signal); } + async callTool(name: string, args: Record, signal?: AbortSignal) { + return await (await this.requireRuntime()).callTool(name, args, signal); + } + async escalateScope(reason: EscalationReason, signal?: AbortSignal) { + return await (await this.requireRuntime()).escalateScope(reason, signal); + } async getScreenSize(signal?: AbortSignal) { return await (await this.requireRuntime()).getScreenSize(signal); } diff --git a/extensions/cua-computer/src/driver-result.ts b/extensions/cua-computer/src/driver-result.ts new file mode 100644 index 000000000000..69a37cd9d1aa --- /dev/null +++ b/extensions/cua-computer/src/driver-result.ts @@ -0,0 +1,364 @@ +import { + COMPUTER_USE_V2_ACTION_NAMES, + type ComputerActResult, + type ComputerUseV2ActionName, +} from "openclaw/plugin-sdk/computer-use"; +import { canonicalizeBase64 } from "openclaw/plugin-sdk/media-runtime"; +import { z } from "zod"; +import type { CuaDriverSession, CuaToolResult } from "./driver-client.js"; +import { + adoptGeneration, + issueAppRef, + issueElementRef, + issueObservation, + issueWindowRef, + type CuaFrameState, +} from "./frame.js"; + +const CUA_WIRE_ACTION_NAMES = COMPUTER_USE_V2_ACTION_NAMES.slice(1, 14); +const CUA_COMMON_ACTION_NAMES = [ + "screenshot", + ...CUA_WIRE_ACTION_NAMES.filter((action) => action !== "hold_key"), + "list_apps", + "list_windows", + "get_accessibility_tree", + "get_cursor_position", + "get_window_state", + "launch_app", + "kill_app", + "bring_to_front", + "set_value", + "zoom", + "escalate_scope", + "invoke_menu", +] as const; + +const NativeAppSchema = z.object({ + pid: z.number().int().nonnegative().nullable().optional(), + bundle_id: z.string().nullable().optional(), + name: z.string().min(1), + running: z.boolean().nullable().optional(), + active: z.boolean().optional(), + kind: z.string().nullable().optional(), + launch_path: z.string().nullable().optional(), + last_used: z.string().nullable().optional(), +}); +const NativeBoundsSchema = z.object({ + x: z.number(), + y: z.number(), + width: z.number().nonnegative(), + height: z.number().nonnegative(), +}); +const NativeWindowSchema = z.object({ + window_id: z.number().int().nonnegative(), + pid: z.number().int().positive().nullable().optional(), + app_name: z.string().optional(), + title: z.string().optional(), + bounds: NativeBoundsSchema, + is_on_screen: z.boolean().optional(), + minimized: z.boolean().optional(), + z_index: z.number().int().nullable().optional(), +}); +const NativeElementSchema = z.object({ + element_index: z.number().int().nonnegative(), + element_token: z.string().min(1).optional(), + role: z.string().optional(), + label: z.string().optional(), + value: z.string().optional(), + frame: z + .object({ + x: z.number(), + y: z.number(), + w: z.number().nonnegative(), + h: z.number().nonnegative(), + }) + .optional(), +}); +const MAX_DISCOVERY_ITEMS = 500; +const PARTIAL_EFFECT = 1 as import("@trycua/cua-driver").ActionEffect; +const VALUE_READBACK_EVIDENCE = 0 as import("@trycua/cua-driver").ActionEvidenceKind; + +export function platformActions(platform: NodeJS.Platform): ComputerUseV2ActionName[] { + return CUA_COMMON_ACTION_NAMES.filter( + (action) => + platform === "linux" || (action !== "left_mouse_down" && action !== "left_mouse_up"), + ) as ComputerUseV2ActionName[]; +} + +function boundedItems(items: T[]): { items: T[]; truncated: number } { + return { + items: items.slice(0, MAX_DISCOVERY_ITEMS), + truncated: Math.max(0, items.length - MAX_DISCOVERY_ITEMS), + }; +} + +function driverEffect(result: CuaToolResult): ComputerActResult["effect"] | undefined { + switch (Number(result.action?.effect)) { + case 0: + return "confirmed"; + case 1: + case 2: + return "unverifiable"; + case 3: + return "suspected_noop"; + case 4: + throw new Error("COMPUTER_REFUSED_action_refused: CUA Driver refused the action"); + default: + return undefined; + } +} + +function driverEscalation(result: CuaToolResult): ComputerActResult["escalation"] | undefined { + const escalation = result.action?.escalation; + if (!escalation) { + return undefined; + } + const recommended = { + 0: "window-pixel", + 1: "foreground", + 2: "window-pixel", + 3: "desktop", + }[escalation.target] as NonNullable["recommended"] | undefined; + const reasonCode = { + 0: "route_unavailable", + 1: "delivery_failed", + 2: "effect_unconfirmed", + 3: "suspected_noop", + 4: "permission_required", + }[escalation.reason]; + if (!recommended || !reasonCode) { + throw new Error("COMPUTER_DRIVER_ERROR: invalid CUA Driver action escalation"); + } + return { recommended, reasonCode }; +} + +function driverActionDetails(result: CuaToolResult): Record | undefined { + const action = result.action; + if (!action) { + return undefined; + } + const details: Record = { + route: [ + "accessibility", + "synthetic_events", + "global_input", + "system_api", + "dom", + "trusted_input", + ][action.route], + }; + if (action.effect === PARTIAL_EFFECT) { + details.partial = true; + } + if (action.delivery) { + details.deliveryMode = ["background", "foreground", "not_applicable", "unknown"][ + action.delivery.mode + ]; + if (action.delivery.deliveredCount !== undefined) { + details.deliveredCount = action.delivery.deliveredCount; + } + } + if (action.evidence?.length) { + details.evidence = action.evidence.map(({ kind }) => + kind === VALUE_READBACK_EVIDENCE ? "value_readback" : "window_change", + ); + } + return Object.values(details).some((value) => value !== undefined) ? details : undefined; +} + +export function actionEnvelope( + result: CuaToolResult, + details?: Record, +): ComputerActResult { + const effect = driverEffect(result); + const escalation = driverEscalation(result); + const driverDetails = driverActionDetails(result); + return { + ok: true, + ...(effect ? { effect } : {}), + ...(escalation ? { escalation } : {}), + ...(driverDetails || details ? { details: { ...driverDetails, ...details } } : {}), + }; +} + +export async function callWindowTool( + driver: CuaDriverSession, + state: CuaFrameState, + name: string, + args: Record, + signal?: AbortSignal, +): Promise { + const result = await driver.callTool(name, args, signal); + adoptGeneration(state, driver.generation); + if (result.isError) { + const code = result.errorCode + ? `COMPUTER_REFUSED_${result.errorCode}` + : "COMPUTER_DRIVER_ERROR"; + throw new Error(`${code}: ${result.text || `${name} failed`}`); + } + return result; +} + +export function projectedToolDetails(result: CuaToolResult, tool: string): Record { + if (!result.structuredJson) { + throw new Error(`COMPUTER_DRIVER_ERROR: ${tool} returned no structuredContent`); + } + try { + const value: unknown = JSON.parse(result.structuredJson); + if (value && typeof value === "object" && !Array.isArray(value)) { + return value as Record; + } + } catch {} + throw new Error(`COMPUTER_DRIVER_ERROR: ${tool} returned invalid structuredContent`); +} + +export function nativeWindows(value: unknown): Array> { + if (!Array.isArray(value)) { + return []; + } + return value.flatMap((entry) => { + const parsed = NativeWindowSchema.safeParse(entry); + return parsed.success && parsed.data.pid ? [parsed.data] : []; + }); +} + +export function projectWindows( + state: CuaFrameState, + windows: Array>, +): { windows: Array>; truncatedWindows?: number } { + const bounded = boundedItems(windows); + return { + windows: bounded.items.map((window) => ({ + windowRef: issueWindowRef(state, { pid: window.pid!, windowId: window.window_id }), + ...(window.app_name ? { appName: window.app_name } : {}), + ...(window.title ? { title: window.title } : {}), + bounds: window.bounds, + ...(window.is_on_screen !== undefined ? { isOnScreen: window.is_on_screen } : {}), + ...(window.minimized !== undefined ? { minimized: window.minimized } : {}), + ...(window.z_index !== undefined ? { zIndex: window.z_index } : {}), + })), + ...(bounded.truncated ? { truncatedWindows: bounded.truncated } : {}), + }; +} + +export function projectApps(state: CuaFrameState, value: unknown): Record { + const raw = Array.isArray(value) ? value : []; + const apps = raw.flatMap((entry) => { + const parsed = NativeAppSchema.safeParse(entry); + if (!parsed.success) { + return []; + } + const app = issueAppRef(state, { + ...(parsed.data.pid ? { pid: parsed.data.pid } : {}), + name: parsed.data.name, + ...(parsed.data.bundle_id ? { bundleId: parsed.data.bundle_id } : {}), + ...(parsed.data.launch_path ? { launchPath: parsed.data.launch_path } : {}), + }); + return [ + { + app, + name: parsed.data.name, + ...(parsed.data.running !== undefined ? { running: parsed.data.running } : {}), + ...(parsed.data.active !== undefined ? { active: parsed.data.active } : {}), + ...(parsed.data.kind ? { kind: parsed.data.kind } : {}), + ...(parsed.data.last_used ? { lastUsed: parsed.data.last_used } : {}), + }, + ]; + }); + const bounded = boundedItems(apps); + return { + apps: bounded.items, + totalApps: apps.length, + ...(bounded.truncated ? { truncatedApps: bounded.truncated } : {}), + }; +} + +export function projectProcesses(value: unknown): Record { + const processes = boundedItems(Array.isArray(value) ? value : []); + return { + processes: processes.items, + ...(processes.truncated ? { truncatedProcesses: processes.truncated } : {}), + }; +} + +export function windowObservation( + result: CuaToolResult, + state: CuaFrameState, + windowRef: string, + options: { fromZoom?: boolean } = {}, +): ComputerActResult { + const structured = projectedToolDetails(result, options.fromZoom ? "zoom" : "get_window_state"); + const observation = issueObservation(state, windowRef, options); + const snapshotId = + typeof structured.snapshot_id === "string" ? structured.snapshot_id : undefined; + const rawElements = Array.isArray(structured.elements) ? structured.elements : []; + let omittedElementCount = 0; + const elements = rawElements.slice(0, 2_000).flatMap((entry) => { + const parsed = NativeElementSchema.safeParse(entry); + if (!parsed.success || !parsed.data.frame) { + omittedElementCount += 1; + return []; + } + const elementRef = issueElementRef(observation, { + elementIndex: parsed.data.element_index, + ...(parsed.data.element_token ? { elementToken: parsed.data.element_token } : {}), + ...(snapshotId ? { snapshotId } : {}), + }); + return [ + { + elementRef, + role: parsed.data.role?.trim() || "unknown", + ...(parsed.data.label !== undefined ? { label: parsed.data.label } : {}), + ...(parsed.data.value !== undefined ? { value: parsed.data.value } : {}), + bounds: { + x: parsed.data.frame.x, + y: parsed.data.frame.y, + width: parsed.data.frame.w, + height: parsed.data.frame.h, + }, + }, + ]; + }); + const image = result.images.find((entry) => entry.mimeType === "image/png"); + const base64 = image ? canonicalizeBase64(image.dataBase64) : undefined; + if (image && !base64) { + throw new Error("COMPUTER_DRIVER_ERROR: CUA Driver returned malformed window PNG base64"); + } + const width = + typeof structured.screenshot_width === "number" && structured.screenshot_width > 0 + ? Math.trunc(structured.screenshot_width) + : undefined; + const height = + typeof structured.screenshot_height === "number" && structured.screenshot_height > 0 + ? Math.trunc(structured.screenshot_height) + : undefined; + const details: Record = { + ...(typeof structured.total_element_count === "number" + ? { totalElementCount: structured.total_element_count } + : {}), + ...(rawElements.length > 2_000 ? { truncatedElements: rawElements.length - 2_000 } : {}), + ...(omittedElementCount ? { omittedElementsWithoutBounds: omittedElementCount } : {}), + ...(structured.degraded === true ? { degraded: true } : {}), + ...(typeof structured.degraded_reason === "string" + ? { degradedReason: structured.degraded_reason } + : {}), + ...(typeof structured.screenshot_error === "string" + ? { screenshotError: structured.screenshot_error } + : {}), + }; + const action = actionEnvelope(result, details); + return { + ...action, + observation: { + kind: "window", + ...(base64 ? { base64, format: "png" as const } : {}), + ...(width ? { width } : {}), + ...(height ? { height } : {}), + observationId: observation.id, + ...(elements.length ? { elements } : {}), + }, + ...(!action.escalation && structured.escalation && typeof structured.escalation === "object" + ? { escalation: { recommended: "window-pixel", reasonCode: "ax_tree_unavailable" } } + : {}), + }; +} diff --git a/extensions/cua-computer/src/frame.ts b/extensions/cua-computer/src/frame.ts index 47d70a68fadb..45b31a2b1f06 100644 --- a/extensions/cua-computer/src/frame.ts +++ b/extensions/cua-computer/src/frame.ts @@ -1,4 +1,4 @@ -import { createHash } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; export type CuaDesktopGeometry = { platform: string; @@ -28,12 +28,147 @@ export type CuaLastFrame = { export type CuaFrameState = { generation: string; lastFrame?: CuaLastFrame; + apps?: Map; + windows?: Map; + observation?: CuaObservationState; +}; + +type CuaAppTarget = { + pid?: number; + name: string; + bundleId?: string; + launchPath?: string; +}; + +type CuaWindowTarget = { + pid: number; + windowId: number; +}; + +type CuaElementTarget = { + elementIndex: number; + elementToken?: string; + snapshotId?: string; +}; + +type CuaObservationState = { + id: string; + windowRef: string; + fromZoom: boolean; + elements: Map; }; function staleFrame(message: string): Error { return new Error(`COMPUTER_STALE_FRAME: ${message}; take a new screenshot`); } +function staleObservation(): Error { + return new Error("COMPUTER_STALE_OBSERVATION: take a fresh observation and retry"); +} + +function opaqueRef(kind: "app" | "window" | "observation" | "element"): string { + return `cua:v2:${kind}:${randomUUID()}`; +} + +export function adoptGeneration(state: CuaFrameState, generation: string): void { + // Native session replacement invalidates every authority-bearing reference, + // even when the same window ids and display geometry reappear. + if (state.generation !== generation) { + state.lastFrame = undefined; + state.apps = undefined; + state.windows = undefined; + state.observation = undefined; + } + state.generation = generation; +} + +export function verifyGeneration(state: CuaFrameState, generation: string): void { + if (state.generation !== generation) { + adoptGeneration(state, generation); + throw staleObservation(); + } +} + +export function issueAppRef(state: CuaFrameState, target: CuaAppTarget): string { + state.apps ??= new Map(); + const ref = opaqueRef("app"); + state.apps.set(ref, target); + return ref; +} + +export function resolveAppRef(state: CuaFrameState, ref: string): CuaAppTarget | undefined { + return state.apps?.get(ref); +} + +export function issueWindowRef(state: CuaFrameState, target: CuaWindowTarget): string { + state.windows ??= new Map(); + for (const [ref, current] of state.windows) { + if (current.pid === target.pid && current.windowId === target.windowId) { + return ref; + } + } + const ref = opaqueRef("window"); + state.windows.set(ref, target); + return ref; +} + +export function resolveWindowRef(state: CuaFrameState, ref: string): CuaWindowTarget { + const target = state.windows?.get(ref); + if (!target) { + throw staleObservation(); + } + return target; +} + +export function issueObservation( + state: CuaFrameState, + windowRef: string, + options: { fromZoom?: boolean } = {}, +): CuaObservationState { + // Only the newest observation may authorize element or window-pixel actions; + // retaining older element tokens would bypass the driver's snapshot lifecycle. + const observation: CuaObservationState = { + id: opaqueRef("observation"), + windowRef, + fromZoom: options.fromZoom === true, + elements: new Map(), + }; + state.observation = observation; + return observation; +} + +export function issueElementRef( + observation: CuaObservationState, + target: CuaElementTarget, +): string { + const ref = opaqueRef("element"); + observation.elements.set(ref, target); + return ref; +} + +export function resolveObservation( + state: CuaFrameState, + observationId: string, + windowRef: string, +): CuaObservationState { + const observation = state.observation; + if (!observation || observation.id !== observationId || observation.windowRef !== windowRef) { + throw staleObservation(); + } + return observation; +} + +export function resolveElementRef( + observation: CuaObservationState, + elementRef: string, +): CuaElementTarget { + const target = observation.elements.get(elementRef); + if (!target) { + throw staleObservation(); + } + return target; +} + /** * CUA Driver exposes only the primary-display label, not a stable display ID. * Bind authorization to connection generation plus the complete live geometry. diff --git a/extensions/cua-computer/src/v2-actions.ts b/extensions/cua-computer/src/v2-actions.ts new file mode 100644 index 000000000000..4024f0ccccfe --- /dev/null +++ b/extensions/cua-computer/src/v2-actions.ts @@ -0,0 +1,524 @@ +import { + COMPUTER_USE_V2_ACTION_NAMES, + type ComputerActParams, +} from "openclaw/plugin-sdk/computer-use"; +import { normalizeModifiers, parseKeyChord } from "./actions.js"; +import { EscalationReason, type CuaDriverSession } from "./driver-client.js"; +import { + actionEnvelope, + callWindowTool, + nativeWindows, + projectApps, + projectedToolDetails, + projectProcesses, + projectWindows, + windowObservation, +} from "./driver-result.js"; +import { + adoptGeneration, + resolveAppRef, + resolveElementRef, + resolveObservation, + resolveWindowRef, + verifyGeneration, + type CuaFrameState, +} from "./frame.js"; + +const CUA_WIRE_ACTION_NAMES = COMPUTER_USE_V2_ACTION_NAMES.slice(1, 14); +const CUA_TARGETED_ACTION_NAMES = new Set([ + "left_click", + "right_click", + "middle_click", + "double_click", + "triple_click", + "left_click_drag", + "left_mouse_down", + "left_mouse_up", + "scroll", + "type", + "key", +] as const); + +export type CuaComputerActParams = { + action: ComputerActParams["action"]; + displayFrameId?: string; + x?: number; + y?: number; + fromX?: number; + fromY?: number; + text?: string; + keys?: string; + modifiers?: string; + scrollDirection?: "up" | "down" | "left" | "right"; + scrollAmount?: number; + durationMs?: number; + screenIndex?: number; + refWidth?: number; + windowRef?: string; + elementRef?: string; + observationId?: string; + deliveryMode?: "background" | "foreground"; + query?: string; + depth?: number; + maxElements?: number; + app?: string; + value?: string; + path?: string[]; + x1?: number; + y1?: number; + x2?: number; + y2?: number; + reason?: + | "ax_tree_pixel_mismatch" + | "background_delivery_failed" + | "foreground_ineffective" + | "no_window_target" + | "other"; +}; + +function requireWindowTarget( + driver: CuaDriverSession, + state: CuaFrameState, + params: CuaComputerActParams, +) { + verifyGeneration(state, driver.generation); + if (!params.windowRef) { + throw new Error(`COMPUTER_INVALID_REQUEST: windowRef is required for ${params.action}`); + } + return { + ref: params.windowRef, + target: resolveWindowRef(state, params.windowRef), + }; +} + +function observationTarget(state: CuaFrameState, params: CuaComputerActParams, windowRef: string) { + if (!params.observationId) { + throw new Error(`COMPUTER_STALE_OBSERVATION: observationId is required for ${params.action}`); + } + return resolveObservation(state, params.observationId, windowRef); +} + +function elementArgs( + state: CuaFrameState, + params: CuaComputerActParams, + windowRef: string, +): Record | undefined { + if (!params.elementRef) { + return undefined; + } + const observation = observationTarget(state, params, windowRef); + const element = resolveElementRef(observation, params.elementRef); + return element.elementToken + ? { element_token: element.elementToken } + : { + element_index: element.elementIndex, + ...(element.snapshotId ? { snapshot_id: element.snapshotId } : {}), + }; +} + +function windowPointArgs( + state: CuaFrameState, + params: CuaComputerActParams, + windowRef: string, + point: { x?: number; y?: number }, + label: string, +): Record { + if (point.x === undefined || point.y === undefined) { + throw new Error(`COMPUTER_INVALID_REQUEST: ${label} coordinates are required`); + } + const observation = observationTarget(state, params, windowRef); + return { + x: point.x, + y: point.y, + ...(observation.fromZoom ? { from_zoom: true } : {}), + }; +} + +async function handleTargetedAct( + platform: NodeJS.Platform, + driver: CuaDriverSession, + state: CuaFrameState, + params: CuaComputerActParams, + signal?: AbortSignal, +): Promise { + const { ref: windowRef, target } = requireWindowTarget(driver, state, params); + const base = { pid: target.pid, window_id: target.windowId }; + const delivery = params.deliveryMode ? { delivery_mode: params.deliveryMode } : {}; + const element = elementArgs(state, params, windowRef); + let tool: string; + let args: Record; + + switch (params.action) { + case "left_click": + case "right_click": + case "middle_click": + case "double_click": + case "triple_click": { + tool = "click"; + const button = + params.action === "right_click" + ? "right" + : params.action === "middle_click" + ? "middle" + : "left"; + const count = params.action === "double_click" ? 2 : params.action === "triple_click" ? 3 : 1; + const modifiers = normalizeModifiers(params.modifiers); + args = { + ...base, + ...(element ?? windowPointArgs(state, params, windowRef, params, "click")), + button, + count, + ...(modifiers.length ? { modifier: modifiers } : {}), + ...delivery, + }; + break; + } + case "left_click_drag": { + if (element) { + throw new Error("COMPUTER_UNSUPPORTED_ACTION: cua-driver drag has no element target"); + } + tool = "drag"; + const from = windowPointArgs( + state, + params, + windowRef, + { x: params.fromX, y: params.fromY }, + "drag start", + ); + const to = windowPointArgs(state, params, windowRef, params, "drag end"); + const modifiers = normalizeModifiers(params.modifiers); + args = { + ...base, + from_x: from.x, + from_y: from.y, + to_x: to.x, + to_y: to.y, + ...(from.from_zoom || to.from_zoom ? { from_zoom: true } : {}), + ...(params.durationMs === undefined + ? {} + : { duration_ms: Math.min(10_000, params.durationMs) }), + ...(modifiers.length ? { modifier: modifiers } : {}), + ...delivery, + }; + break; + } + case "left_mouse_down": { + if (platform !== "linux") { + throw new Error("COMPUTER_UNSUPPORTED_ACTION: left_mouse_down is Linux-only"); + } + if (element || params.deliveryMode === "foreground") { + throw new Error( + "COMPUTER_UNSUPPORTED_ACTION: left_mouse_down supports only background window pixels", + ); + } + tool = "mouse_button_down"; + args = { + ...base, + ...windowPointArgs(state, params, windowRef, params, "mouse down"), + button: "left", + }; + break; + } + case "left_mouse_up": { + if (platform !== "linux") { + throw new Error("COMPUTER_UNSUPPORTED_ACTION: left_mouse_up is Linux-only"); + } + if (element || params.deliveryMode === "foreground") { + throw new Error( + "COMPUTER_UNSUPPORTED_ACTION: left_mouse_up supports only background window pixels", + ); + } + tool = "mouse_button_up"; + args = { + ...base, + ...(params.x !== undefined || params.y !== undefined + ? windowPointArgs(state, params, windowRef, params, "mouse up") + : {}), + }; + break; + } + case "scroll": { + if (!params.scrollDirection) { + throw new Error("COMPUTER_INVALID_REQUEST: scrollDirection is required for scroll"); + } + if (normalizeModifiers(params.modifiers).length) { + throw new Error( + "COMPUTER_UNSUPPORTED_ACTION: modifier-held scroll is unsupported by cua-driver", + ); + } + tool = "scroll"; + args = { + ...base, + ...(element ?? + (params.x !== undefined || params.y !== undefined + ? windowPointArgs(state, params, windowRef, params, "scroll") + : {})), + direction: params.scrollDirection, + by: "line", + amount: Math.min(50, params.scrollAmount ?? 3), + ...delivery, + }; + break; + } + case "type": { + if (!params.text) { + throw new Error("COMPUTER_INVALID_REQUEST: text is required for type"); + } + tool = "type_text"; + args = { + ...base, + ...(element ?? + (params.x !== undefined || params.y !== undefined + ? windowPointArgs(state, params, windowRef, params, "type") + : {})), + text: params.text, + ...delivery, + }; + break; + } + case "key": { + const chord = parseKeyChord(params.keys); + tool = "press_key"; + args = { + ...base, + ...(element ?? + (params.x !== undefined || params.y !== undefined + ? windowPointArgs(state, params, windowRef, params, "key") + : {})), + key: chord.key, + modifiers: chord.modifiers, + ...delivery, + }; + break; + } + default: + throw new Error(`COMPUTER_UNSUPPORTED_ACTION: ${params.action}`); + } + + const result = await callWindowTool(driver, state, tool, args, signal); + return JSON.stringify(actionEnvelope(result)); +} + +export async function handleV2Act( + platform: NodeJS.Platform, + driver: CuaDriverSession, + state: CuaFrameState, + params: ComputerActParams, + handleDesktop: ( + driver: CuaDriverSession, + state: CuaFrameState, + params: ComputerActParams, + signal?: AbortSignal, + ) => Promise, + signal?: AbortSignal, +): Promise { + const input = params as CuaComputerActParams & Record; + if ( + CUA_TARGETED_ACTION_NAMES.has(input.action as never) && + (input.windowRef || input.elementRef) + ) { + return await handleTargetedAct(platform, driver, state, input, signal); + } + if ((CUA_WIRE_ACTION_NAMES as readonly string[]).includes(input.action)) { + return await handleDesktop(driver, state, params, signal); + } + + switch (input.action) { + case "list_apps": { + const result = await callWindowTool(driver, state, "list_apps", {}, signal); + state.apps = new Map(); + const structured = projectedToolDetails(result, "list_apps"); + return JSON.stringify({ ok: true, details: projectApps(state, structured.apps) }); + } + case "list_windows": { + const result = await callWindowTool(driver, state, "list_windows", {}, signal); + const structured = projectedToolDetails(result, "list_windows"); + return JSON.stringify({ + ok: true, + details: projectWindows(state, nativeWindows(structured.windows)), + }); + } + case "get_accessibility_tree": { + if (input.windowRef || input.query || input.depth !== undefined || input.maxElements) { + throw new Error( + "COMPUTER_UNSUPPORTED_ACTION: CUA Driver 0.19.3 exposes get_accessibility_tree only as unfiltered desktop discovery; use get_window_state for a window tree", + ); + } + const result = await callWindowTool(driver, state, "get_accessibility_tree", {}, signal); + const structured = projectedToolDetails(result, "get_accessibility_tree"); + return JSON.stringify({ + ok: true, + details: { + ...projectWindows(state, nativeWindows(structured.windows)), + ...projectProcesses(structured.processes), + }, + }); + } + case "get_cursor_position": { + const result = await callWindowTool(driver, state, "get_cursor_position", {}, signal); + return JSON.stringify({ + ok: true, + details: projectedToolDetails(result, "get_cursor_position"), + }); + } + case "get_window_state": { + verifyGeneration(state, driver.generation); + const window = resolveWindowRef(state, input.windowRef!); + const result = await callWindowTool( + driver, + state, + "get_window_state", + { + pid: window.pid, + window_id: window.windowId, + include_screenshot: true, + max_elements: input.maxElements ?? 2_000, + ...(input.depth !== undefined ? { max_depth: Math.max(1, input.depth) } : {}), + ...(input.query ? { query: input.query } : {}), + }, + signal, + ); + return JSON.stringify(windowObservation(result, state, input.windowRef!)); + } + case "launch_app": { + verifyGeneration(state, driver.generation); + const appName = input.app!; + const app = resolveAppRef(state, appName); + if (appName.startsWith("cua:v2:app:") && !app) { + throw new Error("COMPUTER_STALE_OBSERVATION: refresh list_apps and retry"); + } + const result = await callWindowTool( + driver, + state, + "launch_app", + app + ? app.launchPath + ? { launch_path: app.launchPath } + : app.bundleId + ? { bundle_id: app.bundleId } + : { name: app.name } + : { name: appName }, + signal, + ); + const structured = projectedToolDetails(result, "launch_app"); + return JSON.stringify({ + ...actionEnvelope(result), + details: { + app: projectApps(state, [structured]).apps, + ...projectWindows(state, nativeWindows(structured.windows)), + }, + }); + } + case "kill_app": { + verifyGeneration(state, driver.generation); + const appName = input.app!; + const app = resolveAppRef(state, appName); + if (!app?.pid) { + throw new Error( + "COMPUTER_INVALID_REQUEST: kill_app requires a running app reference from list_apps", + ); + } + const result = await callWindowTool(driver, state, "kill_app", { pid: app.pid }, signal); + return JSON.stringify(actionEnvelope(result, { app: appName })); + } + case "bring_to_front": { + const { target } = requireWindowTarget(driver, state, input); + const result = await callWindowTool( + driver, + state, + "bring_to_front", + { + pid: target.pid, + window_id: target.windowId, + }, + signal, + ); + return JSON.stringify(actionEnvelope(result)); + } + case "set_value": { + const { ref, target } = requireWindowTarget(driver, state, input); + if (input.deliveryMode === "foreground") { + throw new Error( + "COMPUTER_UNSUPPORTED_ACTION: cua-driver set_value is background accessibility delivery", + ); + } + const element = elementArgs(state, input, ref); + if (!element) { + throw new Error("COMPUTER_INVALID_REQUEST: elementRef is required for set_value"); + } + const result = await callWindowTool( + driver, + state, + "set_value", + { + pid: target.pid, + window_id: target.windowId, + ...element, + value: input.value, + }, + signal, + ); + return JSON.stringify(actionEnvelope(result)); + } + case "invoke_menu": { + const { target } = requireWindowTarget(driver, state, input); + if (input.deliveryMode === "foreground") { + throw new Error( + "COMPUTER_UNSUPPORTED_ACTION: cua-driver invoke_menu is background accessibility delivery", + ); + } + const result = await callWindowTool( + driver, + state, + "invoke_menu", + { + pid: target.pid, + window_id: target.windowId, + path: input.path, + }, + signal, + ); + return JSON.stringify(actionEnvelope(result)); + } + case "zoom": { + const { ref, target } = requireWindowTarget(driver, state, input); + resolveObservation(state, input.observationId!, ref); + const result = await callWindowTool( + driver, + state, + "zoom", + { + pid: target.pid, + window_id: target.windowId, + x1: input.x1, + y1: input.y1, + x2: input.x2, + y2: input.y2, + }, + signal, + ); + return JSON.stringify(windowObservation(result, state, ref, { fromZoom: true })); + } + case "escalate_scope": { + const reason = { + ax_tree_pixel_mismatch: EscalationReason.AxTreePixelMismatch, + background_delivery_failed: EscalationReason.BackgroundDeliveryFailed, + foreground_ineffective: EscalationReason.ForegroundIneffective, + no_window_target: EscalationReason.NoWindowTarget, + other: EscalationReason.Other, + }[input.reason!]; + const result = await driver.escalateScope(reason, signal); + adoptGeneration(state, driver.generation); + return JSON.stringify({ + ok: true, + details: { + captureScope: result.captureScope, + effectiveScope: result.effectiveScope, + desktopUnlocked: result.desktopUnlocked, + }, + }); + } + default: + throw new Error(`COMPUTER_UNSUPPORTED_ACTION: ${input.action}`); + } +}