From aa7a2fbe0dda294ed77ee062e67208ade5b486ea Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 14 Aug 2026 20:59:46 -0700 Subject: [PATCH] feat(cua-computer): browser action family over v2 (#123960) * feat(cua-computer): add browser action family * refactor(cua-computer): split browser action files * refactor(cua-computer): move the shared act-params type to the leaf module * fix(cua-computer): carry the contract params import with the moved type --- docs/nodes/computer-use.md | 11 +- extensions/cua-computer/src/action-targets.ts | 172 +++++++ .../cua-computer/src/browser-actions.test.ts | 423 ++++++++++++++++++ .../cua-computer/src/browser-actions.ts | 237 ++++++++++ .../cua-computer/src/commands.test-helpers.ts | 141 ++++++ extensions/cua-computer/src/commands.test.ts | 158 +------ extensions/cua-computer/src/commands.ts | 4 +- .../src/cua-driver-contract.test-fixtures.ts | 80 ++++ .../cua-computer/src/driver-client.test.ts | 27 ++ extensions/cua-computer/src/driver-result.ts | 344 +++++++++++++- extensions/cua-computer/src/frame.ts | 184 +++++++- .../src/mcp-driver-client.test.ts | 28 ++ extensions/cua-computer/src/v2-actions.ts | 109 +---- src/agents/tools/computer-tool.test.ts | 68 ++- src/agents/tools/computer-tool.ts | 196 +++++++- src/plugins/computer-use-contract.test.ts | 39 +- src/plugins/computer-use-contract.ts | 109 ++++- 17 files changed, 2068 insertions(+), 262 deletions(-) create mode 100644 extensions/cua-computer/src/action-targets.ts create mode 100644 extensions/cua-computer/src/browser-actions.test.ts create mode 100644 extensions/cua-computer/src/browser-actions.ts create mode 100644 extensions/cua-computer/src/commands.test-helpers.ts diff --git a/docs/nodes/computer-use.md b/docs/nodes/computer-use.md index 19969a62782d..2aa76fb35152 100644 --- a/docs/nodes/computer-use.md +++ b/docs/nodes/computer-use.md @@ -36,6 +36,8 @@ The built-in `computer` tool takes one action per call. Coordinates are non-nega Providers with the v2 window/element family can additionally expose `list_apps`, `list_windows`, `get_accessibility_tree`, `get_cursor_position`, `get_window_state`, `launch_app`, `kill_app`, `bring_to_front`, `set_value`, `zoom`, `escalate_scope`, and `invoke_menu`. The provider descriptor is authoritative; unavailable actions are omitted rather than emulated through another provider. +The CUA provider also exposes the v2 browser family: `get_browser_state`, `browser_prepare`, `browser_navigate`, `browser_click`, `browser_type`, `browser_dialog`, `browser_set_input_files`, `browser_download`, and `browser_pointer`. Bind a discovered native browser window with `get_browser_state`, then use the returned opaque `browserRef`, `pageRef`, observation, and element references. These references belong to one Computer Use execution and driver generation; navigation invalidates page-element observations, and a driver restart invalidates the complete browser reference set. + Modifier keys ride the `text` field on click and scroll actions (`shift`, `ctrl`, `alt`, `cmd`). After an input action the tool returns a fresh screenshot so the model can observe the result. If more than one computer-capable node is connected, pass `node` explicitly. Screenshots are kept **model-only**: they are never auto-delivered to the chat channel. Treat all on-screen content as untrusted input; the tool warns the model not to follow on-screen instructions that conflict with the user's request. @@ -52,7 +54,13 @@ The app waits until the private socket accepts connections before advertising CU The embedded CUA daemon runs in unrestricted mode because bounded CUA grants require exact launch-time resources and cannot represent OpenClaw's runtime-discovered windows and elements. OpenClaw command arming, pairing approval, and tool policy are the authoritative authorization gate, identical to the shipped Peekaboo fulfiller. The app owns the daemon and its macOS TCC identity, and the daemon accepts local connections only through an owner-only socket directory. -The CUA descriptor advertises window and element targets, background and foreground delivery, screenshots, and accessibility observations. Peekaboo remains the default in this release and advertises the existing coordinate-action family; its v2 adapter is separate work. +The CUA descriptor advertises window, element, and browser targets; background and foreground delivery; and image, accessibility, and browser observations. Peekaboo remains the default in this release and advertises only the action families its native adapter implements. + +#### Browser profiles + +`browser_prepare` can launch a separate driver-owned Chromium process with a new ephemeral profile or a named isolated profile. It never modifies, copies, terminates, or attaches to the selected browser's existing profile. Existing-profile/CDP attachment remains unavailable because it requires the driver's protected embedding-host consent and revocation path; Gateway approval and `computer.act` arming do not substitute for that local consent. + +Browser targets, pages, page elements, and dialogs are opaque capabilities. Retake browser state after navigation, reconnect, or a stale-reference refusal. The adapter never returns provider-native CDP target IDs, tab IDs, or page refs to the model. ### Windows and Linux (experimental, direct SDK) @@ -93,6 +101,7 @@ The `cua-computer` fulfiller surfaces typed error codes in the tool result and n | `COMPUTER_DRIVER_UNAVAILABLE` | The CUA runtime cannot initialize, the macOS app-owned endpoint is absent, or the desktop permissions/session are unavailable. | On macOS, verify CUA is selected and the bundled driver is ready; on Windows/Linux, run `openclaw node run` inside the interactive desktop session. Reinstall OpenClaw if the pinned runtime is missing. | | `COMPUTER_REFUSED_` | The driver refused the action with a structured code such as `background_unavailable`, `background_occluded`, or `foreground_unavailable` (KDE/KWin Wayland). | Bring the target window forward, switch to X11, or use a supported compositor. See the compatibility notes above. | | `COMPUTER_STALE_FRAME` | The coordinates referenced a screenshot that is no longer current (context compaction, a display geometry change, or a reference-width change). | Take a fresh `screenshot` before the coordinate action. | +| `COMPUTER_STALE_OBSERVATION` | A window or browser reference belongs to an older observation, navigation, execution, or driver generation. | Run `get_window_state` or `get_browser_state` again and retry with the new opaque references. | | `COMPUTER_UNSUPPORTED_ACTION` | An action this fulfiller cannot faithfully deliver: `hold_key`, `left_mouse_down`, `left_mouse_up`, or modifier-held click/drag/scroll. | Use a supported action. The typed CUA Driver desktop contract has no held-input or modifier argument for these calls. | | `COMPUTER_UNSUPPORTED_DISPLAY` | A non-primary `screenIndex`, a capture/screen geometry mismatch, or a cursor outside the primary display. | Drive the primary display only. | | `COMPUTER_UNSUPPORTED_KEY` | A `key` value the driver cannot reproduce reliably: a digit or punctuation key whose shift state is layout-dependent, or an unknown key. | Send that text through the `type` action instead. | diff --git a/extensions/cua-computer/src/action-targets.ts b/extensions/cua-computer/src/action-targets.ts new file mode 100644 index 000000000000..f91158a6bf8e --- /dev/null +++ b/extensions/cua-computer/src/action-targets.ts @@ -0,0 +1,172 @@ +import type { ComputerActParams } from "openclaw/plugin-sdk/computer-use"; +import type { CuaDriverSession } from "./driver-client.js"; +import { + resolveBrowserElementRef, + resolveBrowserObservation, + resolveBrowserRef, + resolveElementRef, + resolveObservation, + resolvePageRef, + resolveWindowRef, + verifyGeneration, + type CuaFrameState, +} from "./frame.js"; + +export type CuaComputerActParams = { + action: ComputerActParams["action"]; + displayFrameId?: string; + x?: number; + y?: number; + fromX?: number; + fromY?: number; + text?: string; + keys?: string; + modifiers?: string; + scrollDirection?: "up" | "down" | "left" | "right"; + scrollAmount?: number; + durationMs?: number; + screenIndex?: number; + refWidth?: number; + windowRef?: string; + elementRef?: string; + observationId?: string; + deliveryMode?: "background" | "foreground"; + query?: string; + depth?: number; + maxElements?: number; + app?: string; + value?: string; + path?: string[]; + browserRef?: string; + pageRef?: string; + snapshotFormat?: "dom_refs_v1" | "semantic_v2"; + continuation?: string; + includeScreenshot?: boolean; + profile?: "isolated_new" | "isolated_named"; + profileName?: string; + url?: string; + inputRoute?: "trusted" | "dom_event"; + mode?: "insert_text" | "keystrokes"; + replace?: boolean; + dialogAction?: "inspect" | "accept" | "dismiss"; + dialogRef?: string; + promptText?: string; + files?: string[]; + destinationRoot?: string; + pointerAction?: "hover" | "right_click" | "double_click" | "scroll" | "drag"; + destinationElementRef?: string; + toX?: number; + toY?: number; + deltaX?: number; + deltaY?: number; + x1?: number; + y1?: number; + x2?: number; + y2?: number; + reason?: + | "ax_tree_pixel_mismatch" + | "background_delivery_failed" + | "foreground_ineffective" + | "no_window_target" + | "other"; +}; + +export function requireWindowTarget( + driver: CuaDriverSession, + state: CuaFrameState, + params: CuaComputerActParams, +) { + verifyGeneration(state, driver.generation); + if (!params.windowRef) { + throw new Error(`COMPUTER_INVALID_REQUEST: windowRef is required for ${params.action}`); + } + return { + ref: params.windowRef, + target: resolveWindowRef(state, params.windowRef), + }; +} + +function observationTarget(state: CuaFrameState, params: CuaComputerActParams, windowRef: string) { + if (!params.observationId) { + throw new Error(`COMPUTER_STALE_OBSERVATION: observationId is required for ${params.action}`); + } + return resolveObservation(state, params.observationId, windowRef); +} + +export function elementArgs( + state: CuaFrameState, + params: CuaComputerActParams, + windowRef: string, +): Record | undefined { + if (!params.elementRef) { + return undefined; + } + const observation = observationTarget(state, params, windowRef); + const element = resolveElementRef(observation, params.elementRef); + return element.elementToken + ? { element_token: element.elementToken } + : { + element_index: element.elementIndex, + ...(element.snapshotId ? { snapshot_id: element.snapshotId } : {}), + }; +} + +export function browserTarget( + driver: CuaDriverSession, + state: CuaFrameState, + params: CuaComputerActParams, +) { + verifyGeneration(state, driver.generation); + if (!params.browserRef || !params.pageRef) { + throw new Error( + `COMPUTER_INVALID_REQUEST: browserRef and pageRef are required for ${params.action}`, + ); + } + const browser = resolveBrowserRef(state, params.browserRef); + const page = resolvePageRef(state, params.browserRef, params.pageRef); + return { + browserRef: params.browserRef, + pageRef: params.pageRef, + targetId: browser.targetId, + tabId: page.tabId, + }; +} + +export function browserElement( + state: CuaFrameState, + params: CuaComputerActParams, + target: { browserRef: string; pageRef: string }, + elementRef = params.elementRef, +): string | undefined { + if (!elementRef) { + return undefined; + } + if (!params.observationId) { + throw new Error(`COMPUTER_STALE_OBSERVATION: observationId is required for ${params.action}`); + } + const observation = resolveBrowserObservation( + state, + params.observationId, + target.browserRef, + target.pageRef, + ); + return resolveBrowserElementRef(observation, elementRef); +} + +export function windowPointArgs( + state: CuaFrameState, + params: CuaComputerActParams, + windowRef: string, + point: { x?: number; y?: number }, + label: string, +): Record { + if (point.x === undefined || point.y === undefined) { + throw new Error(`COMPUTER_INVALID_REQUEST: ${label} coordinates are required`); + } + const observation = observationTarget(state, params, windowRef); + return { + x: point.x, + y: point.y, + ...(observation.fromZoom ? { from_zoom: true } : {}), + }; +} diff --git a/extensions/cua-computer/src/browser-actions.test.ts b/extensions/cua-computer/src/browser-actions.test.ts new file mode 100644 index 000000000000..a19705344f04 --- /dev/null +++ b/extensions/cua-computer/src/browser-actions.test.ts @@ -0,0 +1,423 @@ +import { describe, expect, it } from "vitest"; +import { driver, execution } from "./commands.test-helpers.js"; +import { + CUA_DRIVER_CONTRACT_FIXTURES, + cuaToolResult, +} from "./cua-driver-contract.test-fixtures.js"; +import type { CuaToolResult } from "./driver-client.js"; + +describe("cua-computer browser actions", () => { + it("maps every browser action to the pinned driver tool contract", async () => { + const { session, callTool } = driver(); + callTool.mockImplementation(async (name, args) => { + switch (name) { + case "list_windows": + return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.listWindows); + case "get_browser_state": + return cuaToolResult( + "target_id" in args + ? CUA_DRIVER_CONTRACT_FIXTURES.browserSnapshot + : CUA_DRIVER_CONTRACT_FIXTURES.browserBinding, + { image: "target_id" in args }, + ); + case "browser_prepare": + return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.browserPrepare); + case "browser_navigate": + return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.browserNavigate); + case "browser_dialog": + return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.browserDialog); + case "browser_set_input_files": + return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.browserFiles); + case "browser_download": + return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.browserDownload); + case "browser_click": + case "browser_type": + case "browser_pointer": + return cuaToolResult( + {}, + { + action: + CUA_DRIVER_CONTRACT_FIXTURES.confirmedBackgroundAction as unknown as CuaToolResult["action"], + }, + ); + default: + return cuaToolResult({}); + } + }); + const computer = await execution(session); + const listed = JSON.parse(await computer.act('{"action":"list_windows"}')) as { + details: { windows: Array<{ windowRef: string }> }; + }; + const windowRef = listed.details.windows[0]!.windowRef; + + await computer.act( + JSON.stringify({ + action: "browser_prepare", + windowRef, + profile: "isolated_named", + profileName: "openclaw-test", + }), + ); + const boundJson = await computer.act( + JSON.stringify({ action: "get_browser_state", windowRef }), + ); + expect(boundJson).not.toContain("native-browser-target-1"); + expect(boundJson).not.toContain("native-page-1"); + const bound = JSON.parse(boundJson) as { + details: { browserRef: string; pages: Array<{ pageRef: string }> }; + }; + expect(bound.details.browserRef).toMatch(/^cua:v2:browser:/); + expect(bound.details.pages[0]!.pageRef).toMatch(/^cua:v2:page:/); + const browserRef = bound.details.browserRef; + const pageRef = bound.details.pages[0]!.pageRef; + + const observedJson = await computer.act( + JSON.stringify({ action: "get_browser_state", browserRef, pageRef }), + ); + expect(observedJson).not.toContain("p7:0"); + const observed = JSON.parse(observedJson) as { + observation: { kind: string; observationId: string }; + details: { elements: Array<{ elementRef: string }> }; + }; + expect(observed.observation.kind).toBe("browser"); + const observationId = observed.observation.observationId; + const [firstElement, secondElement] = observed.details.elements.map( + (element) => element.elementRef, + ); + expect(firstElement).toMatch(/^cua:v2:element:/); + + await computer.act( + JSON.stringify({ + action: "browser_click", + browserRef, + pageRef, + observationId, + elementRef: firstElement, + inputRoute: "dom_event", + }), + ); + await computer.act( + JSON.stringify({ + action: "browser_type", + browserRef, + pageRef, + observationId, + elementRef: secondElement, + text: "hello", + mode: "keystrokes", + replace: true, + }), + ); + const dialog = JSON.parse( + await computer.act( + JSON.stringify({ + action: "browser_dialog", + browserRef, + pageRef, + dialogAction: "inspect", + }), + ), + ) as { details: { dialogRef: string } }; + expect(dialog.details.dialogRef).toMatch(/^cua:v2:dialog:/); + await computer.act( + JSON.stringify({ + action: "browser_set_input_files", + browserRef, + pageRef, + observationId, + elementRef: secondElement, + files: ["/tmp/input.txt"], + }), + ); + await computer.act( + JSON.stringify({ + action: "browser_download", + browserRef, + pageRef, + observationId, + elementRef: firstElement, + destinationRoot: "/tmp/downloads", + }), + ); + await computer.act( + JSON.stringify({ + action: "browser_pointer", + browserRef, + pageRef, + observationId, + pointerAction: "drag", + inputRoute: "dom_event", + elementRef: firstElement, + destinationElementRef: secondElement, + }), + ); + await computer.act( + JSON.stringify({ + action: "browser_navigate", + browserRef, + pageRef, + url: "https://example.com/next", + }), + ); + + expect(callTool.mock.calls).toEqual([ + ["list_windows", {}, undefined], + [ + "browser_prepare", + { + pid: 4242, + allow_launch: true, + profile: { mode: "isolated_named", name: "openclaw-test" }, + }, + undefined, + ], + ["get_browser_state", { pid: 4242, window_id: 99 }, undefined], + [ + "get_browser_state", + { + target_id: "native-browser-target-1", + tab_id: "native-page-1", + snapshot_format: "dom_refs_v1", + include_screenshot: true, + }, + undefined, + ], + [ + "browser_click", + { + target_id: "native-browser-target-1", + tab_id: "native-page-1", + ref: "p7:0", + input_route: "dom_event", + }, + undefined, + ], + [ + "browser_type", + { + target_id: "native-browser-target-1", + tab_id: "native-page-1", + ref: "p7:1", + text: "hello", + mode: "keystrokes", + replace: true, + }, + undefined, + ], + [ + "browser_dialog", + { + target_id: "native-browser-target-1", + tab_id: "native-page-1", + action: "inspect", + }, + undefined, + ], + [ + "browser_set_input_files", + { + target_id: "native-browser-target-1", + tab_id: "native-page-1", + ref: "p7:1", + files: ["/tmp/input.txt"], + }, + undefined, + ], + [ + "browser_download", + { + target_id: "native-browser-target-1", + tab_id: "native-page-1", + ref: "p7:0", + destination_root: "/tmp/downloads", + }, + undefined, + ], + [ + "browser_pointer", + { + target_id: "native-browser-target-1", + tab_id: "native-page-1", + action: "drag", + input_route: "dom_event", + ref: "p7:0", + destination_ref: "p7:1", + }, + undefined, + ], + [ + "browser_navigate", + { + target_id: "native-browser-target-1", + tab_id: "native-page-1", + url: "https://example.com/next", + }, + undefined, + ], + ]); + }); + + it("invalidates browser capabilities across navigation, generation, and execution", async () => { + const first = driver(); + first.callTool.mockImplementation(async (name, args) => { + if (name === "list_windows") { + return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.listWindows); + } + if (name === "get_browser_state") { + return cuaToolResult( + "target_id" in args + ? CUA_DRIVER_CONTRACT_FIXTURES.browserSnapshot + : CUA_DRIVER_CONTRACT_FIXTURES.browserBinding, + ); + } + return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.browserNavigate); + }); + const computer = await execution(first.session); + const listed = JSON.parse(await computer.act('{"action":"list_windows"}')) as { + details: { windows: Array<{ windowRef: string }> }; + }; + const bound = JSON.parse( + await computer.act( + JSON.stringify({ + action: "get_browser_state", + windowRef: listed.details.windows[0]!.windowRef, + }), + ), + ) as { details: { browserRef: string; pages: Array<{ pageRef: string }> } }; + const browserRef = bound.details.browserRef; + const pageRef = bound.details.pages[0]!.pageRef; + const observed = JSON.parse( + await computer.act(JSON.stringify({ action: "get_browser_state", browserRef, pageRef })), + ) as { + observation: { observationId: string }; + details: { elements: Array<{ elementRef: string }> }; + }; + const staleAction = { + action: "browser_click", + browserRef, + pageRef, + observationId: observed.observation.observationId, + elementRef: observed.details.elements[0]!.elementRef, + }; + + await computer.act( + JSON.stringify({ action: "browser_navigate", browserRef, pageRef, url: "about:blank" }), + ); + await expect(computer.act(JSON.stringify(staleAction))).rejects.toThrow( + "COMPUTER_STALE_OBSERVATION", + ); + + first.setGeneration("execution-2"); + await expect( + computer.act(JSON.stringify({ action: "get_browser_state", browserRef, pageRef })), + ).rejects.toThrow("COMPUTER_STALE_OBSERVATION"); + + const second = await execution(first.session); + await expect( + second.act(JSON.stringify({ action: "get_browser_state", browserRef, pageRef })), + ).rejects.toThrow("COMPUTER_STALE_OBSERVATION"); + }); + + it("keeps existing-profile browser attachment outside the accepted contract", async () => { + const { session, callTool } = driver(); + callTool.mockResolvedValueOnce(cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.listWindows)); + const computer = await execution(session); + const listed = JSON.parse(await computer.act('{"action":"list_windows"}')) as { + details: { windows: Array<{ windowRef: string }> }; + }; + const windowRef = listed.details.windows[0]!.windowRef; + + await expect( + computer.act( + JSON.stringify({ + action: "browser_prepare", + windowRef, + profile: "existing_profile", + }), + ), + ).rejects.toThrow("COMPUTER_INVALID_REQUEST"); + await expect( + computer.act( + JSON.stringify({ + action: "browser_prepare", + windowRef, + strategy: { kind: "existing_profile" }, + }), + ), + ).rejects.toThrow("COMPUTER_INVALID_REQUEST"); + expect(callTool).toHaveBeenCalledTimes(1); + }); + + it("rechecks browser generation and structured stale refusals after driver calls", async () => { + const active = driver(); + let staleOnSnapshot = true; + active.callTool.mockImplementation(async (name, args) => { + if (name === "list_windows") { + return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.listWindows); + } + if (name === "get_browser_state" && !("target_id" in args)) { + return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.browserBinding); + } + if (name === "get_browser_state" && staleOnSnapshot) { + active.setGeneration("execution-2"); + return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.browserSnapshot); + } + if (name === "get_browser_state") { + return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.browserSnapshot); + } + return cuaToolResult({ + status: "refused", + refusal: { code: "browser_ref_stale", message: "page changed" }, + }); + }); + const computer = await execution(active.session); + const listed = JSON.parse(await computer.act('{"action":"list_windows"}')) as { + details: { windows: Array<{ windowRef: string }> }; + }; + const bind = async () => + JSON.parse( + await computer.act( + JSON.stringify({ + action: "get_browser_state", + windowRef: listed.details.windows[0]!.windowRef, + }), + ), + ) as { details: { browserRef: string; pages: Array<{ pageRef: string }> } }; + const firstBinding = await bind(); + await expect( + computer.act( + JSON.stringify({ + action: "get_browser_state", + browserRef: firstBinding.details.browserRef, + pageRef: firstBinding.details.pages[0]!.pageRef, + }), + ), + ).rejects.toThrow("COMPUTER_STALE_OBSERVATION"); + + staleOnSnapshot = false; + const refreshedWindows = JSON.parse(await computer.act('{"action":"list_windows"}')) as { + details: { windows: Array<{ windowRef: string }> }; + }; + listed.details.windows = refreshedWindows.details.windows; + const secondBinding = await bind(); + const browserRef = secondBinding.details.browserRef; + const pageRef = secondBinding.details.pages[0]!.pageRef; + const observed = JSON.parse( + await computer.act(JSON.stringify({ action: "get_browser_state", browserRef, pageRef })), + ) as { + observation: { observationId: string }; + details: { elements: Array<{ elementRef: string }> }; + }; + await expect( + computer.act( + JSON.stringify({ + action: "browser_click", + browserRef, + pageRef, + observationId: observed.observation.observationId, + elementRef: observed.details.elements[0]!.elementRef, + }), + ), + ).rejects.toThrow("COMPUTER_STALE_OBSERVATION"); + }); +}); diff --git a/extensions/cua-computer/src/browser-actions.ts b/extensions/cua-computer/src/browser-actions.ts new file mode 100644 index 000000000000..cb5df49053d5 --- /dev/null +++ b/extensions/cua-computer/src/browser-actions.ts @@ -0,0 +1,237 @@ +import { browserElement, browserTarget, requireWindowTarget } from "./action-targets.js"; +import type { CuaComputerActParams } from "./action-targets.js"; +import type { CuaDriverSession } from "./driver-client.js"; +import { + browserBinding, + browserDialogEnvelope, + browserObservation, + browserToolEnvelope, + callWindowTool, +} from "./driver-result.js"; +import { + clearDialogRef, + invalidateBrowserObservation, + resolveBrowserObservation, + resolveDialogRef, + resolveWindowRef, + verifyGeneration, + type CuaFrameState, +} from "./frame.js"; + +export async function handleBrowserAct( + driver: CuaDriverSession, + state: CuaFrameState, + input: CuaComputerActParams, + signal?: AbortSignal, +): Promise { + switch (input.action) { + case "get_browser_state": { + verifyGeneration(state, driver.generation); + if (input.windowRef) { + const window = resolveWindowRef(state, input.windowRef); + const result = await callWindowTool( + driver, + state, + "get_browser_state", + { pid: window.pid, window_id: window.windowId }, + signal, + ); + return JSON.stringify(browserBinding(result, state, input.windowRef)); + } + const target = browserTarget(driver, state, input); + const snapshotFormat = input.snapshotFormat ?? "dom_refs_v1"; + if ( + snapshotFormat === "dom_refs_v1" && + (input.elementRef || input.query || input.continuation) + ) { + throw new Error( + "COMPUTER_INVALID_REQUEST: elementRef, query, and continuation require snapshotFormat=semantic_v2", + ); + } + const scopeRef = browserElement(state, input, target); + const result = await callWindowTool( + driver, + state, + "get_browser_state", + { + target_id: target.targetId, + tab_id: target.tabId, + snapshot_format: snapshotFormat, + include_screenshot: input.includeScreenshot ?? true, + ...(scopeRef ? { scope_ref: scopeRef } : {}), + ...(input.query ? { query: input.query } : {}), + ...(input.continuation ? { continuation: input.continuation } : {}), + }, + signal, + ); + return JSON.stringify(browserObservation(result, state, target)); + } + case "browser_prepare": { + const { target } = requireWindowTarget(driver, state, input); + const profile = input.profile ?? "isolated_new"; + if (profile === "isolated_named" && !input.profileName) { + throw new Error( + "COMPUTER_INVALID_REQUEST: profileName is required for an isolated_named browser profile", + ); + } + if (profile === "isolated_new" && input.profileName) { + throw new Error( + "COMPUTER_INVALID_REQUEST: profileName is valid only for an isolated_named browser profile", + ); + } + const result = await callWindowTool( + driver, + state, + "browser_prepare", + { + pid: target.pid, + allow_launch: true, + profile: { + mode: profile, + ...(input.profileName ? { name: input.profileName } : {}), + }, + }, + signal, + ); + return JSON.stringify(browserToolEnvelope(result, "browser_prepare")); + } + case "browser_navigate": { + const target = browserTarget(driver, state, input); + const result = await callWindowTool( + driver, + state, + "browser_navigate", + { target_id: target.targetId, tab_id: target.tabId, url: input.url }, + signal, + ); + invalidateBrowserObservation(state); + return JSON.stringify(browserToolEnvelope(result, "browser_navigate")); + } + case "browser_click": { + const target = browserTarget(driver, state, input); + resolveBrowserObservation(state, input.observationId!, target.browserRef, target.pageRef); + const ref = browserElement(state, input, target); + const result = await callWindowTool( + driver, + state, + "browser_click", + { + target_id: target.targetId, + tab_id: target.tabId, + ...(ref ? { ref } : {}), + ...(input.x !== undefined ? { x: input.x } : {}), + ...(input.y !== undefined ? { y: input.y } : {}), + ...(input.inputRoute ? { input_route: input.inputRoute } : {}), + }, + signal, + ); + return JSON.stringify(browserToolEnvelope(result, "browser_click")); + } + case "browser_type": { + const target = browserTarget(driver, state, input); + const ref = browserElement(state, input, target)!; + const result = await callWindowTool( + driver, + state, + "browser_type", + { + target_id: target.targetId, + tab_id: target.tabId, + ref, + text: input.text, + ...(input.mode ? { mode: input.mode } : {}), + ...(input.replace !== undefined ? { replace: input.replace } : {}), + }, + signal, + ); + return JSON.stringify(browserToolEnvelope(result, "browser_type")); + } + case "browser_dialog": { + const target = browserTarget(driver, state, input); + const dialogId = + input.dialogAction === "inspect" + ? undefined + : resolveDialogRef(state, input.dialogRef!, target.browserRef, target.pageRef); + const result = await callWindowTool( + driver, + state, + "browser_dialog", + { + target_id: target.targetId, + tab_id: target.tabId, + action: input.dialogAction, + ...(dialogId ? { dialog_id: dialogId } : {}), + ...(input.promptText !== undefined ? { prompt_text: input.promptText } : {}), + ...(input.deliveryMode ? { delivery_mode: input.deliveryMode } : {}), + }, + signal, + ); + if (input.dialogAction !== "inspect") { + clearDialogRef(state); + } + return JSON.stringify(browserDialogEnvelope(result, state, target)); + } + case "browser_set_input_files": { + const target = browserTarget(driver, state, input); + const ref = browserElement(state, input, target)!; + const result = await callWindowTool( + driver, + state, + "browser_set_input_files", + { + target_id: target.targetId, + tab_id: target.tabId, + ref, + files: input.files, + }, + signal, + ); + return JSON.stringify(browserToolEnvelope(result, "browser_set_input_files")); + } + case "browser_download": { + const target = browserTarget(driver, state, input); + const ref = browserElement(state, input, target)!; + const result = await callWindowTool( + driver, + state, + "browser_download", + { + target_id: target.targetId, + tab_id: target.tabId, + ref, + destination_root: input.destinationRoot, + }, + signal, + ); + return JSON.stringify(browserToolEnvelope(result, "browser_download")); + } + case "browser_pointer": { + const target = browserTarget(driver, state, input); + resolveBrowserObservation(state, input.observationId!, target.browserRef, target.pageRef); + const ref = browserElement(state, input, target); + const destinationRef = browserElement(state, input, target, input.destinationElementRef); + const result = await callWindowTool( + driver, + state, + "browser_pointer", + { + target_id: target.targetId, + tab_id: target.tabId, + action: input.pointerAction, + ...(input.inputRoute ? { input_route: input.inputRoute } : {}), + ...(ref ? { ref } : {}), + ...(input.x !== undefined ? { x: input.x } : {}), + ...(input.y !== undefined ? { y: input.y } : {}), + ...(destinationRef ? { destination_ref: destinationRef } : {}), + ...(input.toX !== undefined ? { to_x: input.toX } : {}), + ...(input.toY !== undefined ? { to_y: input.toY } : {}), + ...(input.deltaX !== undefined ? { delta_x: input.deltaX } : {}), + ...(input.deltaY !== undefined ? { delta_y: input.deltaY } : {}), + }, + signal, + ); + return JSON.stringify(browserToolEnvelope(result, "browser_pointer")); + } + } + return undefined; +} diff --git a/extensions/cua-computer/src/commands.test-helpers.ts b/extensions/cua-computer/src/commands.test-helpers.ts new file mode 100644 index 000000000000..81568d5c6b0f --- /dev/null +++ b/extensions/cua-computer/src/commands.test-helpers.ts @@ -0,0 +1,141 @@ +import { vi } from "vitest"; +import { createCuaComputerProvider } from "./commands.js"; +import type { CuaDriverSession, CuaToolResult } from "./driver-client.js"; + +const geometry = { + platform: "linux", + display: "primary", + screenshot_width: 100, + screenshot_height: 50, + screen_width: 100, + screen_height: 50, + scale_factor: 1, +}; + +const CUA_DRIVER_ENDPOINT_ENV = "OPENCLAW_CUA_DRIVER_ENDPOINT"; + +export function macOsEndpoint(overrides: Record = {}): NodeJS.ProcessEnv { + return { + [CUA_DRIVER_ENDPOINT_ENV]: JSON.stringify({ + v: 1, + socketPath: "/tmp/openclaw-cua-test/driver.sock", + binaryPath: process.execPath, + ...overrides, + }), + }; +} + +export function invalidMacOsEndpoints(): Array<[string, NodeJS.ProcessEnv]> { + return [ + ["missing", {}], + ["malformed JSON", { [CUA_DRIVER_ENDPOINT_ENV]: "{" }], + [ + "partial", + { + [CUA_DRIVER_ENDPOINT_ENV]: JSON.stringify({ + v: 1, + socketPath: "/tmp/openclaw-cua-test/driver.sock", + }), + }, + ], + ["unsupported version", macOsEndpoint({ v: 2 })], + ["extra field", macOsEndpoint({ extra: true })], + ["relative socket", macOsEndpoint({ socketPath: "relative.sock" })], + ["relative binary", macOsEndpoint({ binaryPath: "cua-driver" })], + ["nul socket", macOsEndpoint({ socketPath: "/tmp/cua\0.sock" })], + ["missing binary", macOsEndpoint({ binaryPath: "/missing/cua-driver" })], + ["oversized", macOsEndpoint({ socketPath: `/${"x".repeat(4_096)}` })], + ]; +} + +export function result(structured: Record, image = false): CuaToolResult { + return { + text: "ok", + images: image + ? [{ mimeType: "image/png", dataBase64: Buffer.from("png").toString("base64") }] + : [], + structuredJson: JSON.stringify(structured), + isError: false, + degraded: false, + rawJson: "{}", + }; +} + +export function driver( + options: { + geometry?: typeof geometry; + screenSize?: { width: number; height: number; scale_factor: number }; + } = {}, +) { + let generation = "execution-1"; + const activeGeometry = options.geometry ?? geometry; + const getDesktopState = vi.fn(async () => result(activeGeometry, true)); + const getScreenSize = vi.fn(async () => + result( + options.screenSize ?? { + width: activeGeometry.screen_width, + height: activeGeometry.screen_height, + scale_factor: activeGeometry.scale_factor, + }, + ), + ); + const click = vi.fn(async () => result({})); + const drag = vi.fn(async () => result({})); + const moveCursor = vi.fn(async () => result({})); + const scroll = vi.fn(async () => result({})); + const typeText = vi.fn(async () => result({})); + const pressKey = vi.fn(async () => result({})); + const callTool = vi.fn(async () => result({})); + const escalateScope = vi.fn(async () => ({ + session: "openclaw-test", + captureScope: 2, + effectiveScope: 1, + desktopUnlocked: true, + })); + const dispose = vi.fn(async () => {}); + const session: CuaDriverSession = { + get generation() { + return generation; + }, + isAvailable: () => true, + resetAvailabilityCache: () => {}, + callTool, + escalateScope, + getDesktopState, + getScreenSize, + click, + drag, + moveCursor, + scroll, + typeText, + pressKey, + dispose, + }; + return { + session, + getDesktopState, + getScreenSize, + click, + drag, + moveCursor, + scroll, + callTool, + escalateScope, + dispose, + typeText, + pressKey, + setGeneration: (value: string) => { + generation = value; + }, + }; +} + +export async function execution(session: CuaDriverSession) { + return await createCuaComputerProvider({ + platform: "linux", + driver: session, + imageProcessor: { + encode: vi.fn(async () => ({ data: Buffer.from("jpeg"), width: 100, height: 50 })), + }, + }).openExecution({}); +} diff --git a/extensions/cua-computer/src/commands.test.ts b/extensions/cua-computer/src/commands.test.ts index 3da61fa86efb..75c0b4a786f5 100644 --- a/extensions/cua-computer/src/commands.test.ts +++ b/extensions/cua-computer/src/commands.test.ts @@ -1,5 +1,12 @@ import { describe, expect, it, vi } from "vitest"; import { createCuaComputerProvider } from "./commands.js"; +import { + driver, + execution, + invalidMacOsEndpoints, + macOsEndpoint, + result, +} from "./commands.test-helpers.js"; import { CUA_DRIVER_CONTRACT_FIXTURES, cuaToolResult, @@ -8,125 +15,9 @@ import { ClickButton, EscalationReason, ScrollDirection, - type CuaDriverSession, type CuaToolResult, } from "./driver-client.js"; -const geometry = { - platform: "linux", - display: "primary", - screenshot_width: 100, - screenshot_height: 50, - screen_width: 100, - screen_height: 50, - scale_factor: 1, -}; - -const CUA_DRIVER_ENDPOINT_ENV = "OPENCLAW_CUA_DRIVER_ENDPOINT"; - -function macOsEndpoint(overrides: Record = {}): NodeJS.ProcessEnv { - return { - [CUA_DRIVER_ENDPOINT_ENV]: JSON.stringify({ - v: 1, - socketPath: "/tmp/openclaw-cua-test/driver.sock", - binaryPath: process.execPath, - ...overrides, - }), - }; -} - -function result(structured: Record, image = false): CuaToolResult { - return { - text: "ok", - images: image - ? [{ mimeType: "image/png", dataBase64: Buffer.from("png").toString("base64") }] - : [], - structuredJson: JSON.stringify(structured), - isError: false, - degraded: false, - rawJson: "{}", - }; -} - -function driver( - options: { - geometry?: typeof geometry; - screenSize?: { width: number; height: number; scale_factor: number }; - } = {}, -) { - let generation = "execution-1"; - const activeGeometry = options.geometry ?? geometry; - const getDesktopState = vi.fn(async () => result(activeGeometry, true)); - const getScreenSize = vi.fn(async () => - result( - options.screenSize ?? { - width: activeGeometry.screen_width, - height: activeGeometry.screen_height, - scale_factor: activeGeometry.scale_factor, - }, - ), - ); - const click = vi.fn(async () => result({})); - const drag = vi.fn(async () => result({})); - const moveCursor = vi.fn(async () => result({})); - const scroll = vi.fn(async () => result({})); - const typeText = vi.fn(async () => result({})); - const pressKey = vi.fn(async () => result({})); - const callTool = vi.fn(async () => result({})); - const escalateScope = vi.fn(async () => ({ - session: "openclaw-test", - captureScope: 2, - effectiveScope: 1, - desktopUnlocked: true, - })); - const dispose = vi.fn(async () => {}); - const session: CuaDriverSession = { - get generation() { - return generation; - }, - isAvailable: () => true, - resetAvailabilityCache: () => {}, - callTool, - escalateScope, - getDesktopState, - getScreenSize, - click, - drag, - moveCursor, - scroll, - typeText, - pressKey, - dispose, - }; - return { - session, - getDesktopState, - getScreenSize, - click, - drag, - moveCursor, - scroll, - callTool, - escalateScope, - dispose, - typeText, - pressKey, - setGeneration: (value: string) => { - generation = value; - }, - }; -} - -async function execution(session: CuaDriverSession) { - return await createCuaComputerProvider({ - platform: "linux", - driver: session, - imageProcessor: { - encode: vi.fn(async () => ({ data: Buffer.from("jpeg"), width: 100, height: 50 })), - }, - }).openExecution({}); -} - describe("cua-computer provider", () => { it("advertises the implemented Linux v2 capability", () => { const { session } = driver(); @@ -165,12 +56,21 @@ describe("cua-computer provider", () => { "bring_to_front", "set_value", "zoom", + "get_browser_state", + "browser_prepare", + "browser_navigate", + "browser_click", + "browser_type", + "browser_dialog", + "browser_set_input_files", + "browser_download", + "browser_pointer", "escalate_scope", "invoke_menu", ], - targets: ["screen", "window", "element"], + targets: ["screen", "window", "element", "browser"], deliveryModes: ["background", "foreground"], - observations: ["image", "accessibility"], + observations: ["image", "accessibility", "browser"], features: { recording: false, agentCursor: false, multiDisplay: false }, }); }); @@ -212,27 +112,7 @@ describe("cua-computer provider", () => { ).toBe(true); expect(createDriver).not.toHaveBeenCalled(); - const invalidEndpoints: Array<[string, NodeJS.ProcessEnv]> = [ - ["missing", {}], - ["malformed JSON", { [CUA_DRIVER_ENDPOINT_ENV]: "{" }], - [ - "partial", - { - [CUA_DRIVER_ENDPOINT_ENV]: JSON.stringify({ - v: 1, - socketPath: "/tmp/openclaw-cua-test/driver.sock", - }), - }, - ], - ["unsupported version", macOsEndpoint({ v: 2 })], - ["extra field", macOsEndpoint({ extra: true })], - ["relative socket", macOsEndpoint({ socketPath: "relative.sock" })], - ["relative binary", macOsEndpoint({ binaryPath: "cua-driver" })], - ["nul socket", macOsEndpoint({ socketPath: "/tmp/cua\0.sock" })], - ["missing binary", macOsEndpoint({ binaryPath: "/missing/cua-driver" })], - ["oversized", macOsEndpoint({ socketPath: `/${"x".repeat(4_096)}` })], - ]; - for (const [label, env] of invalidEndpoints) { + for (const [label, env] of invalidMacOsEndpoints()) { expect( createCuaComputerProvider({ platform: "darwin", env, driver: session }).isAvailable(), label, diff --git a/extensions/cua-computer/src/commands.ts b/extensions/cua-computer/src/commands.ts index 18dc4bdcb4f4..d7bf0bdf41b3 100644 --- a/extensions/cua-computer/src/commands.ts +++ b/extensions/cua-computer/src/commands.ts @@ -482,9 +482,9 @@ export function createCuaComputerProvider( : "cua-computer-v2:unsupported", }, actions: platformActions(platform), - targets: ["screen", "window", "element"], + targets: ["screen", "window", "element", "browser"], deliveryModes: ["background", "foreground"], - observations: ["image", "accessibility"], + observations: ["image", "accessibility", "browser"], features: { recording: false, agentCursor: false, multiDisplay: false }, }), isAvailable, diff --git a/extensions/cua-computer/src/cua-driver-contract.test-fixtures.ts b/extensions/cua-computer/src/cua-driver-contract.test-fixtures.ts index 9fe91e08b15a..85e458b64021 100644 --- a/extensions/cua-computer/src/cua-driver-contract.test-fixtures.ts +++ b/extensions/cua-computer/src/cua-driver-contract.test-fixtures.ts @@ -49,6 +49,86 @@ export const CUA_DRIVER_CONTRACT_FIXTURES = { }, ], }, + browserBinding: { + status: "ok", + mode: "bind", + target_id: "native-browser-target-1", + binding_quality: "exact", + binding_route: "native_cdp_window", + mutation_allowed: true, + native_title: "Example", + tabs: [ + { + tab_id: "native-page-1", + title: "Example", + url: "https://example.com/", + active: true, + }, + ], + }, + browserSnapshot: { + status: "ok", + mode: "snapshot", + target_id: "native-browser-target-1", + tab_id: "native-page-1", + snapshot_id: "p7", + url: "https://example.com/", + refs: [ + { + ref: "p7:0", + node: "BUTTON", + label: "Continue", + frame: "main", + }, + { + ref: "p7:1", + node: "INPUT", + label: "Name", + frame: "main", + }, + ], + truncated: false, + screenshot_width: 1_280, + screenshot_height: 720, + }, + browserPrepare: { + status: "ok", + prepared: true, + action: "launched_isolated_browser", + message: "launched isolated browser", + endpoint_ownership: { method: "spawned_by_driver" }, + prepared_pid: 9001, + side_effects: { launched_browser: true, created_profile: true }, + attachment: null, + }, + browserNavigate: { + status: "ok", + target_id: "native-browser-target-1", + tab_id: "native-page-1", + url: "https://example.com/next", + refs_invalidated: true, + }, + browserDialog: { + status: "ok", + target_id: "native-browser-target-1", + tab_id: "native-page-1", + present: true, + dialog_id: "dialog-4", + kind: "prompt", + }, + browserFiles: { + status: "ok", + target_id: "native-browser-target-1", + tab_id: "native-page-1", + ref: "p7:1", + frame: "main", + file_count: 1, + }, + browserDownload: { + status: "completed", + download_id: "opaque-download-guid", + bytes: 42, + }, confirmedBackgroundAction: { effect: 0, route: 0, diff --git a/extensions/cua-computer/src/driver-client.test.ts b/extensions/cua-computer/src/driver-client.test.ts index fa74c95a8352..1f9090137585 100644 --- a/extensions/cua-computer/src/driver-client.test.ts +++ b/extensions/cua-computer/src/driver-client.test.ts @@ -146,6 +146,33 @@ describe("CUA Driver direct session", () => { await driver.dispose(); }); + it("passes browser tools through the same window-scoped direct SDK session", async () => { + const driver = createCuaDriver({ loadSdk: () => sdk as never }); + + await driver.callTool("browser_navigate", { + target_id: "target-1", + tab_id: "tab-1", + url: "https://example.com/", + }); + const sessionOptions = mocks.createTrustedSession.mock.calls[0]?.[1]; + + expect(mocks.startSession).toHaveBeenCalledWith( + { session: sessionOptions.publicSession, captureScope: "window" }, + undefined, + ); + expect(mocks.callTool).toHaveBeenCalledWith( + "browser_navigate", + JSON.stringify({ + target_id: "target-1", + tab_id: "tab-1", + url: "https://example.com/", + session: sessionOptions.publicSession, + }), + undefined, + ); + await driver.dispose(); + }); + it("keeps a missing native desktop library behind command availability", async () => { const loadSdk = vi.fn(() => { throw new Error("libX11.so.6: cannot open shared object file"); diff --git a/extensions/cua-computer/src/driver-result.ts b/extensions/cua-computer/src/driver-result.ts index 69a37cd9d1aa..d795598378af 100644 --- a/extensions/cua-computer/src/driver-result.ts +++ b/extensions/cua-computer/src/driver-result.ts @@ -8,6 +8,13 @@ import { z } from "zod"; import type { CuaDriverSession, CuaToolResult } from "./driver-client.js"; import { adoptGeneration, + clearDialogRef, + invalidateBrowserReferences, + issueBrowserElementRef, + issueBrowserObservation, + issueBrowserRef, + issueDialogRef, + issuePageRef, issueAppRef, issueElementRef, issueObservation, @@ -29,6 +36,15 @@ const CUA_COMMON_ACTION_NAMES = [ "bring_to_front", "set_value", "zoom", + "get_browser_state", + "browser_prepare", + "browser_navigate", + "browser_click", + "browser_type", + "browser_dialog", + "browser_set_input_files", + "browser_download", + "browser_pointer", "escalate_scope", "invoke_menu", ] as const; @@ -74,7 +90,26 @@ const NativeElementSchema = z.object({ }) .optional(), }); +const NativeBrowserTabSchema = z.object({ + tab_id: z.string().min(1), + title: z.string().optional(), + url: z.string().optional(), + active: z.boolean().optional(), +}); +const NativeBrowserRefSchema = z.object({ + ref: z.string().min(1), + node: z.string().optional(), + role: z.string().optional(), + label: z.string().optional(), + name: z.string().optional(), + value: z.string().optional(), + states: z.array(z.string()).optional(), + actions: z.array(z.string()).optional(), + frame: z.string().optional(), + visibility: z.string().optional(), +}); const MAX_DISCOVERY_ITEMS = 500; +const MAX_BROWSER_ELEMENTS = 2_000; const PARTIAL_EFFECT = 1 as import("@trycua/cua-driver").ActionEffect; const VALUE_READBACK_EVIDENCE = 0 as import("@trycua/cua-driver").ActionEvidenceKind; @@ -188,17 +223,51 @@ export async function callWindowTool( args: Record, signal?: AbortSignal, ): Promise { + const callGeneration = driver.generation; + const stateWasCurrent = state.generation === callGeneration; const result = await driver.callTool(name, args, signal); + if (stateWasCurrent && driver.generation !== callGeneration) { + adoptGeneration(state, driver.generation); + throw new Error("COMPUTER_STALE_OBSERVATION: computer driver generation changed during action"); + } adoptGeneration(state, driver.generation); - if (result.isError) { - const code = result.errorCode - ? `COMPUTER_REFUSED_${result.errorCode}` - : "COMPUTER_DRIVER_ERROR"; + const refusalCode = result.errorCode ?? structuredRefusalCode(result); + if (result.isError || refusalCode) { + if ( + refusalCode && + [ + "browser_binding_stale", + "browser_tab_not_found", + "browser_ref_stale", + "browser_reconnect_exhausted", + ].includes(refusalCode) + ) { + invalidateBrowserReferences(state); + throw new Error("COMPUTER_STALE_OBSERVATION: take a fresh browser observation and retry"); + } + const code = refusalCode ? `COMPUTER_REFUSED_${refusalCode}` : "COMPUTER_DRIVER_ERROR"; throw new Error(`${code}: ${result.text || `${name} failed`}`); } return result; } +function structuredRefusalCode(result: CuaToolResult): string | undefined { + if (!result.structuredJson) { + return undefined; + } + try { + const value = JSON.parse(result.structuredJson) as { + status?: unknown; + refusal?: { code?: unknown }; + }; + return value.status === "refused" && typeof value.refusal?.code === "string" + ? value.refusal.code + : undefined; + } catch { + return undefined; + } +} + export function projectedToolDetails(result: CuaToolResult, tool: string): Record { if (!result.structuredJson) { throw new Error(`COMPUTER_DRIVER_ERROR: ${tool} returned no structuredContent`); @@ -362,3 +431,270 @@ export function windowObservation( : {}), }; } + +export function browserBinding( + result: CuaToolResult, + state: CuaFrameState, + windowRef: string, +): ComputerActResult { + const structured = projectedToolDetails(result, "get_browser_state"); + if ( + structured.mode !== "bind" || + typeof structured.target_id !== "string" || + structured.target_id.length === 0 || + !Array.isArray(structured.tabs) + ) { + throw new Error("COMPUTER_DRIVER_ERROR: invalid browser bind result"); + } + const browserRef = issueBrowserRef(state, { targetId: structured.target_id, windowRef }); + const pages = structured.tabs.flatMap((entry) => { + const parsed = NativeBrowserTabSchema.safeParse(entry); + if (!parsed.success) { + return []; + } + return [ + { + pageRef: issuePageRef(state, browserRef, parsed.data.tab_id), + ...(parsed.data.title !== undefined ? { title: parsed.data.title } : {}), + ...(parsed.data.url !== undefined ? { url: parsed.data.url } : {}), + ...(parsed.data.active !== undefined ? { active: parsed.data.active } : {}), + }, + ]; + }); + const bounded = boundedItems(pages); + return { + ok: true, + details: { + browserRef, + pages: bounded.items, + ...(bounded.truncated ? { truncatedPages: bounded.truncated } : {}), + ...(typeof structured.binding_quality === "string" + ? { bindingQuality: structured.binding_quality } + : {}), + ...(typeof structured.binding_route === "string" + ? { bindingRoute: structured.binding_route } + : {}), + ...(typeof structured.mutation_allowed === "boolean" + ? { mutationAllowed: structured.mutation_allowed } + : {}), + ...(typeof structured.native_title === "string" + ? { nativeTitle: structured.native_title } + : {}), + }, + }; +} + +export function browserObservation( + result: CuaToolResult, + state: CuaFrameState, + target: { + browserRef: string; + pageRef: string; + targetId: string; + tabId: string; + }, +): ComputerActResult { + const structured = projectedToolDetails(result, "get_browser_state"); + if ( + structured.mode !== "snapshot" || + structured.target_id !== target.targetId || + structured.tab_id !== target.tabId + ) { + throw new Error("COMPUTER_DRIVER_ERROR: invalid browser snapshot result"); + } + const observation = issueBrowserObservation(state, target.browserRef, target.pageRef); + const rawRefs = [ + ...(Array.isArray(structured.refs) + ? structured.refs.map((value) => ({ value, kind: "action" })) + : []), + ...(Array.isArray(structured.content_refs) + ? structured.content_refs.map((value) => ({ value, kind: "content" })) + : []), + ]; + const seen = new Set(); + const elements = rawRefs.flatMap(({ value, kind }) => { + const parsed = NativeBrowserRefSchema.safeParse(value); + if (!parsed.success || seen.has(parsed.data.ref)) { + return []; + } + seen.add(parsed.data.ref); + return [ + { + elementRef: issueBrowserElementRef(observation, parsed.data.ref), + kind, + ...(parsed.data.node !== undefined ? { node: parsed.data.node } : {}), + ...(parsed.data.role !== undefined ? { role: parsed.data.role } : {}), + ...(parsed.data.label !== undefined ? { label: parsed.data.label } : {}), + ...(parsed.data.name !== undefined ? { name: parsed.data.name } : {}), + ...(parsed.data.value !== undefined ? { value: parsed.data.value } : {}), + ...(parsed.data.states !== undefined ? { states: parsed.data.states } : {}), + ...(parsed.data.actions !== undefined ? { actions: parsed.data.actions } : {}), + ...(parsed.data.frame !== undefined ? { frame: parsed.data.frame } : {}), + ...(parsed.data.visibility !== undefined ? { visibility: parsed.data.visibility } : {}), + }, + ]; + }); + const boundedElements = elements.slice(0, MAX_BROWSER_ELEMENTS); + const image = result.images.find((entry) => entry.mimeType === "image/png"); + const base64 = image ? canonicalizeBase64(image.dataBase64) : undefined; + if (image && !base64) { + throw new Error("COMPUTER_DRIVER_ERROR: CUA Driver returned malformed browser PNG base64"); + } + const width = + typeof structured.screenshot_width === "number" && structured.screenshot_width > 0 + ? Math.trunc(structured.screenshot_width) + : undefined; + const height = + typeof structured.screenshot_height === "number" && structured.screenshot_height > 0 + ? Math.trunc(structured.screenshot_height) + : undefined; + const page = + structured.page && typeof structured.page === "object" && !Array.isArray(structured.page) + ? (structured.page as Record) + : undefined; + return { + ok: true, + observation: { + kind: "browser", + ...(base64 ? { base64, format: "png" as const } : {}), + ...(width ? { width } : {}), + ...(height ? { height } : {}), + observationId: observation.id, + }, + details: { + browserRef: target.browserRef, + pageRef: target.pageRef, + elements: boundedElements, + ...(elements.length > MAX_BROWSER_ELEMENTS + ? { truncatedElements: elements.length - MAX_BROWSER_ELEMENTS } + : {}), + ...(typeof structured.snapshot_id === "string" + ? { snapshot: { format: "dom_refs_v1" } } + : structured.snapshot && + typeof structured.snapshot === "object" && + !Array.isArray(structured.snapshot) + ? { + snapshot: projectSemanticBrowserSnapshot( + structured.snapshot as Record, + ), + } + : {}), + ...(typeof structured.url === "string" ? { url: structured.url } : {}), + ...(page + ? { + page: { + ...(typeof page.url === "string" ? { url: page.url } : {}), + ...(typeof page.title === "string" ? { title: page.title } : {}), + }, + } + : {}), + ...(typeof structured.truncated === "boolean" ? { truncated: structured.truncated } : {}), + }, + }; +} + +export function browserToolEnvelope( + result: CuaToolResult, + tool: + | "browser_prepare" + | "browser_navigate" + | "browser_click" + | "browser_type" + | "browser_set_input_files" + | "browser_download" + | "browser_pointer", +): ComputerActResult { + if (tool === "browser_click" || tool === "browser_type" || tool === "browser_pointer") { + return actionEnvelope(result); + } + const structured = projectedToolDetails(result, tool); + const details: Record = {}; + if (tool === "browser_prepare") { + for (const [source, destination] of [ + ["prepared", "prepared"], + ["action", "action"], + ["message", "message"], + ["side_effects", "sideEffects"], + ] as const) { + if (structured[source] !== undefined) { + details[destination] = structured[source]; + } + } + const endpointOwnership = structured.endpoint_ownership; + if ( + endpointOwnership && + typeof endpointOwnership === "object" && + !Array.isArray(endpointOwnership) && + typeof (endpointOwnership as Record).method === "string" + ) { + details.endpointOwnership = { + method: (endpointOwnership as Record).method, + }; + } + } else if (tool === "browser_navigate") { + if (typeof structured.url === "string") { + details.url = structured.url; + } + if (typeof structured.refs_invalidated === "boolean") { + details.refsInvalidated = structured.refs_invalidated; + } + } else if (tool === "browser_set_input_files") { + if (typeof structured.file_count === "number") { + details.fileCount = structured.file_count; + } + if (typeof structured.frame === "string") { + details.frame = structured.frame; + } + } else if (tool === "browser_download") { + if (typeof structured.status === "string") { + details.status = structured.status; + } + if (typeof structured.bytes === "number") { + details.bytes = structured.bytes; + } + } + return { ok: true, ...(Object.keys(details).length ? { details } : {}) }; +} + +function projectSemanticBrowserSnapshot(snapshot: Record) { + return { + format: "semantic_v2", + ...(typeof snapshot.complete === "boolean" ? { complete: snapshot.complete } : {}), + ...(typeof snapshot.selected_nodes === "number" + ? { selectedNodes: snapshot.selected_nodes } + : {}), + ...(typeof snapshot.total_nodes === "number" ? { totalNodes: snapshot.total_nodes } : {}), + ...(snapshot.omitted && typeof snapshot.omitted === "object" && !Array.isArray(snapshot.omitted) + ? { omitted: snapshot.omitted } + : {}), + ...(typeof snapshot.continuation === "string" ? { continuation: snapshot.continuation } : {}), + }; +} + +export function browserDialogEnvelope( + result: CuaToolResult, + state: CuaFrameState, + target: { browserRef: string; pageRef: string }, +): ComputerActResult { + const structured = projectedToolDetails(result, "browser_dialog"); + const present = structured.present === true; + if (!present) { + clearDialogRef(state); + } + const details: Record = { present }; + if (typeof structured.kind === "string") { + details.kind = structured.kind; + } + if (present && typeof structured.dialog_id === "string") { + details.dialogRef = issueDialogRef( + state, + structured.dialog_id, + target.browserRef, + target.pageRef, + ); + } + if (typeof structured.action === "string") { + details.action = structured.action; + } + return { ok: true, details }; +} diff --git a/extensions/cua-computer/src/frame.ts b/extensions/cua-computer/src/frame.ts index 45b31a2b1f06..15eed466ef80 100644 --- a/extensions/cua-computer/src/frame.ts +++ b/extensions/cua-computer/src/frame.ts @@ -31,6 +31,10 @@ export type CuaFrameState = { apps?: Map; windows?: Map; observation?: CuaObservationState; + browsers?: Map; + pages?: Map; + browserObservation?: CuaBrowserObservationState; + dialog?: CuaDialogState; }; type CuaAppTarget = { @@ -58,6 +62,34 @@ type CuaObservationState = { elements: Map; }; +type CuaBrowserTarget = { + targetId: string; + windowRef: string; +}; + +type CuaPageTarget = { + browserRef: string; + tabId: string; +}; + +type CuaBrowserElementTarget = { + nativeRef: string; +}; + +type CuaBrowserObservationState = { + id: string; + browserRef: string; + pageRef: string; + elements: Map; +}; + +type CuaDialogState = { + ref: string; + nativeId: string; + browserRef: string; + pageRef: string; +}; + function staleFrame(message: string): Error { return new Error(`COMPUTER_STALE_FRAME: ${message}; take a new screenshot`); } @@ -66,7 +98,9 @@ function staleObservation(): Error { return new Error("COMPUTER_STALE_OBSERVATION: take a fresh observation and retry"); } -function opaqueRef(kind: "app" | "window" | "observation" | "element"): string { +function opaqueRef( + kind: "app" | "window" | "observation" | "element" | "browser" | "page" | "dialog", +): string { return `cua:v2:${kind}:${randomUUID()}`; } @@ -78,6 +112,10 @@ export function adoptGeneration(state: CuaFrameState, generation: string): void state.apps = undefined; state.windows = undefined; state.observation = undefined; + state.browsers = undefined; + state.pages = undefined; + state.browserObservation = undefined; + state.dialog = undefined; } state.generation = generation; } @@ -169,6 +207,150 @@ export function resolveElementRef( return target; } +export function issueBrowserRef(state: CuaFrameState, target: CuaBrowserTarget): string { + state.browsers ??= new Map(); + for (const [ref, current] of state.browsers) { + if (current.targetId === target.targetId && current.windowRef === target.windowRef) { + return ref; + } + } + const ref = opaqueRef("browser"); + state.browsers.set(ref, target); + return ref; +} + +export function resolveBrowserRef(state: CuaFrameState, ref: string): CuaBrowserTarget { + const target = state.browsers?.get(ref); + if (!target) { + throw staleObservation(); + } + return target; +} + +export function issuePageRef(state: CuaFrameState, browserRef: string, tabId: string): string { + state.pages ??= new Map(); + for (const [ref, current] of state.pages) { + if (current.browserRef === browserRef && current.tabId === tabId) { + return ref; + } + } + const ref = opaqueRef("page"); + state.pages.set(ref, { browserRef, tabId }); + return ref; +} + +export function resolvePageRef( + state: CuaFrameState, + browserRef: string, + pageRef: string, +): CuaPageTarget { + const page = state.pages?.get(pageRef); + if (!page || page.browserRef !== browserRef) { + throw staleObservation(); + } + return page; +} + +export function issueBrowserObservation( + state: CuaFrameState, + browserRef: string, + pageRef: string, +): CuaBrowserObservationState { + // CUA invalidates page refs after navigation and each newer snapshot. Keep + // only the newest browser observation so stale DOM capabilities fail closed. + const observation: CuaBrowserObservationState = { + id: opaqueRef("observation"), + browserRef, + pageRef, + elements: new Map(), + }; + state.browserObservation = observation; + state.dialog = undefined; + return observation; +} + +export function issueBrowserElementRef( + observation: CuaBrowserObservationState, + nativeRef: string, +): string { + const ref = opaqueRef("element"); + observation.elements.set(ref, { nativeRef }); + return ref; +} + +export function resolveBrowserObservation( + state: CuaFrameState, + observationId: string, + browserRef: string, + pageRef: string, +): CuaBrowserObservationState { + const observation = state.browserObservation; + if ( + !observation || + observation.id !== observationId || + observation.browserRef !== browserRef || + observation.pageRef !== pageRef + ) { + throw staleObservation(); + } + return observation; +} + +export function resolveBrowserElementRef( + observation: CuaBrowserObservationState, + elementRef: string, +): string { + const target = observation.elements.get(elementRef); + if (!target) { + throw staleObservation(); + } + return target.nativeRef; +} + +export function invalidateBrowserObservation(state: CuaFrameState): void { + state.browserObservation = undefined; + state.dialog = undefined; +} + +export function invalidateBrowserReferences(state: CuaFrameState): void { + state.browsers = undefined; + state.pages = undefined; + invalidateBrowserObservation(state); +} + +export function issueDialogRef( + state: CuaFrameState, + nativeId: string, + browserRef: string, + pageRef: string, +): string { + const ref = opaqueRef("dialog"); + state.dialog = { ref, nativeId, browserRef, pageRef }; + return ref; +} + +export function resolveDialogRef( + state: CuaFrameState, + dialogRef: string, + browserRef: string, + pageRef: string, +): string { + const dialog = state.dialog; + if ( + !dialog || + dialog.ref !== dialogRef || + dialog.browserRef !== browserRef || + dialog.pageRef !== pageRef + ) { + throw staleObservation(); + } + return dialog.nativeId; +} + +export function clearDialogRef(state: CuaFrameState): void { + state.dialog = undefined; +} + /** * CUA Driver exposes only the primary-display label, not a stable display ID. * Bind authorization to connection generation plus the complete live geometry. diff --git a/extensions/cua-computer/src/mcp-driver-client.test.ts b/extensions/cua-computer/src/mcp-driver-client.test.ts index 21f784270ea6..ee9cdaec62ba 100644 --- a/extensions/cua-computer/src/mcp-driver-client.test.ts +++ b/extensions/cua-computer/src/mcp-driver-client.test.ts @@ -177,6 +177,18 @@ describe.runIf(process.platform !== "win32")("CUA MCP proxy transport", () => { }), ); break; + case "browser_navigate": + fake.respond( + request, + toolResult({ + status: "ok", + target_id: "target-1", + tab_id: "tab-1", + url: "https://example.com/", + refs_invalidated: true, + }), + ); + break; case "end_session": fake.respond(request, toolResult({ session: "openclaw-test", active: false })); break; @@ -209,6 +221,11 @@ describe.runIf(process.platform !== "win32")("CUA MCP proxy transport", () => { delivery: { mode: 0, deliveredCount: 1 }, evidence: [{ kind: 0 }], }); + await driver.callTool("browser_navigate", { + target_id: "target-1", + tab_id: "tab-1", + url: "https://example.com/", + }); await driver.dispose(); await vi.waitFor(() => { @@ -225,6 +242,17 @@ describe.runIf(process.platform !== "win32")("CUA MCP proxy transport", () => { (request) => request.method === "tools/call" && request.params?.name === "click", )?.params?.arguments, ).toMatchObject({ x: 20, y: 30, button: "left", count: 1, scope: "desktop" }); + expect( + endpoint.requests.find( + (request) => + request.method === "tools/call" && request.params?.name === "browser_navigate", + )?.params?.arguments, + ).toMatchObject({ + target_id: "target-1", + tab_id: "tab-1", + url: "https://example.com/", + session: expect.stringMatching(/^openclaw-/), + }); } finally { await endpoint.close(); } diff --git a/extensions/cua-computer/src/v2-actions.ts b/extensions/cua-computer/src/v2-actions.ts index 4024f0ccccfe..639bcb37a128 100644 --- a/extensions/cua-computer/src/v2-actions.ts +++ b/extensions/cua-computer/src/v2-actions.ts @@ -2,7 +2,14 @@ import { COMPUTER_USE_V2_ACTION_NAMES, type ComputerActParams, } from "openclaw/plugin-sdk/computer-use"; +import { + elementArgs, + requireWindowTarget, + windowPointArgs, + type CuaComputerActParams, +} from "./action-targets.js"; import { normalizeModifiers, parseKeyChord } from "./actions.js"; +import { handleBrowserAct } from "./browser-actions.js"; import { EscalationReason, type CuaDriverSession } from "./driver-client.js"; import { actionEnvelope, @@ -17,7 +24,6 @@ import { import { adoptGeneration, resolveAppRef, - resolveElementRef, resolveObservation, resolveWindowRef, verifyGeneration, @@ -39,101 +45,6 @@ const CUA_TARGETED_ACTION_NAMES = new Set([ "key", ] as const); -export type CuaComputerActParams = { - action: ComputerActParams["action"]; - displayFrameId?: string; - x?: number; - y?: number; - fromX?: number; - fromY?: number; - text?: string; - keys?: string; - modifiers?: string; - scrollDirection?: "up" | "down" | "left" | "right"; - scrollAmount?: number; - durationMs?: number; - screenIndex?: number; - refWidth?: number; - windowRef?: string; - elementRef?: string; - observationId?: string; - deliveryMode?: "background" | "foreground"; - query?: string; - depth?: number; - maxElements?: number; - app?: string; - value?: string; - path?: string[]; - x1?: number; - y1?: number; - x2?: number; - y2?: number; - reason?: - | "ax_tree_pixel_mismatch" - | "background_delivery_failed" - | "foreground_ineffective" - | "no_window_target" - | "other"; -}; - -function requireWindowTarget( - driver: CuaDriverSession, - state: CuaFrameState, - params: CuaComputerActParams, -) { - verifyGeneration(state, driver.generation); - if (!params.windowRef) { - throw new Error(`COMPUTER_INVALID_REQUEST: windowRef is required for ${params.action}`); - } - return { - ref: params.windowRef, - target: resolveWindowRef(state, params.windowRef), - }; -} - -function observationTarget(state: CuaFrameState, params: CuaComputerActParams, windowRef: string) { - if (!params.observationId) { - throw new Error(`COMPUTER_STALE_OBSERVATION: observationId is required for ${params.action}`); - } - return resolveObservation(state, params.observationId, windowRef); -} - -function elementArgs( - state: CuaFrameState, - params: CuaComputerActParams, - windowRef: string, -): Record | undefined { - if (!params.elementRef) { - return undefined; - } - const observation = observationTarget(state, params, windowRef); - const element = resolveElementRef(observation, params.elementRef); - return element.elementToken - ? { element_token: element.elementToken } - : { - element_index: element.elementIndex, - ...(element.snapshotId ? { snapshot_id: element.snapshotId } : {}), - }; -} - -function windowPointArgs( - state: CuaFrameState, - params: CuaComputerActParams, - windowRef: string, - point: { x?: number; y?: number }, - label: string, -): Record { - if (point.x === undefined || point.y === undefined) { - throw new Error(`COMPUTER_INVALID_REQUEST: ${label} coordinates are required`); - } - const observation = observationTarget(state, params, windowRef); - return { - x: point.x, - y: point.y, - ...(observation.fromZoom ? { from_zoom: true } : {}), - }; -} - async function handleTargetedAct( platform: NodeJS.Platform, driver: CuaDriverSession, @@ -299,6 +210,8 @@ async function handleTargetedAct( return JSON.stringify(actionEnvelope(result)); } +export type { CuaComputerActParams } from "./action-targets.js"; + export async function handleV2Act( platform: NodeJS.Platform, driver: CuaDriverSession, @@ -322,6 +235,10 @@ export async function handleV2Act( if ((CUA_WIRE_ACTION_NAMES as readonly string[]).includes(input.action)) { return await handleDesktop(driver, state, params, signal); } + const browserResult = await handleBrowserAct(driver, state, input, signal); + if (browserResult !== undefined) { + return browserResult; + } switch (input.action) { case "list_apps": { diff --git a/src/agents/tools/computer-tool.test.ts b/src/agents/tools/computer-tool.test.ts index 18eec04f349e..32b5111bd0d5 100644 --- a/src/agents/tools/computer-tool.test.ts +++ b/src/agents/tools/computer-tool.test.ts @@ -63,9 +63,9 @@ function v2Descriptor( contractVersion: 2 as const, provider: { id: "fixture", label: "Fixture", generation: "generation-1" }, actions, - targets: ["screen", "window", "element"] as const, + targets: ["screen", "window", "element", "browser"] as const, deliveryModes: ["background", "foreground"] as const, - observations: ["image", "accessibility"] as const, + observations: ["image", "accessibility", "browser"] as const, features: { recording: false, agentCursor: false, multiDisplay: false }, ...overrides, }; @@ -526,12 +526,70 @@ describe("createComputerTool execution", () => { expect(callGatewayToolMock).not.toHaveBeenCalled(); }); - it("rejects contract-only actions even when a node advertises them", async () => { - const actions: ComputerUseV2ActionName[] = ["browser_click"]; + it("maps browser observations and opaque refs through the public tool", async () => { + const actions: ComputerUseV2ActionName[] = ["get_browser_state", "browser_pointer"]; + listNodesMock.mockResolvedValue([macComputerNode({ computerUse: v2Descriptor(actions) })]); + callGatewayToolMock.mockResolvedValueOnce({ + payload: { + ok: true, + observation: { kind: "browser", observationId: "browser-observation-1" }, + details: { + browserRef: "browser-1", + pageRef: "page-1", + elements: [{ elementRef: "element-1" }, { elementRef: "element-2" }], + }, + }, + }); + const tool = createVisionComputerTool({ capabilityDescriptor: v2Descriptor(actions) }); + + await tool.execute("observe-browser", { + action: "get_browser_state", + browserRef: "browser-1", + pageRef: "page-1", + snapshotFormat: "dom_refs_v1", + includeScreenshot: true, + }); + expect(readLastComputerActParams()).toEqual({ + action: "get_browser_state", + browserRef: "browser-1", + pageRef: "page-1", + snapshotFormat: "dom_refs_v1", + includeScreenshot: true, + }); + + callGatewayToolMock.mockImplementation(async (_method, _opts, body) => + (body as ComputerActBody).command === COMPUTER_ACT_COMMAND + ? { payload: { ok: true, effect: "confirmed" } } + : screenshotPayload(), + ); + await tool.execute("drag-browser", { + action: "browser_pointer", + browserRef: "browser-1", + pageRef: "page-1", + observationId: "browser-observation-1", + pointerAction: "drag", + inputRoute: "dom_event", + elementRef: "element-1", + destinationElementRef: "element-2", + }); + expect(readLastComputerActParams()).toEqual({ + action: "browser_pointer", + browserRef: "browser-1", + pageRef: "page-1", + observationId: "browser-observation-1", + pointerAction: "drag", + inputRoute: "dom_event", + elementRef: "element-1", + destinationElementRef: "element-2", + }); + }); + + it("rejects recording actions that remain contract-only", async () => { + const actions: ComputerUseV2ActionName[] = ["start_recording"]; listNodesMock.mockResolvedValue([macComputerNode({ computerUse: v2Descriptor(actions) })]); const tool = createVisionComputerTool({ capabilityDescriptor: v2Descriptor(actions) }); - await expect(tool.execute("browser", { action: "browser_click" })).rejects.toThrow( + await expect(tool.execute("record", { action: "start_recording" })).rejects.toThrow( "COMPUTER_CONTRACT_MISMATCH", ); expect(callGatewayToolMock).not.toHaveBeenCalled(); diff --git a/src/agents/tools/computer-tool.ts b/src/agents/tools/computer-tool.ts index 0decb0b8ca8d..fe8f78a6121c 100644 --- a/src/agents/tools/computer-tool.ts +++ b/src/agents/tools/computer-tool.ts @@ -164,6 +164,13 @@ function createComputerToolSchema(actions: readonly ComputerUseV2ActionName[]) { description: "left_click_drag: [x, y] drag origin in screenshot pixels.", }), ), + destinationCoordinate: Type.Optional( + Type.Array(Type.Number({ minimum: 0 }), { + minItems: 2, + maxItems: 2, + description: "browser_pointer drag destination [x, y] in viewport CSS pixels.", + }), + ), text: Type.Optional( Type.String({ description: @@ -191,6 +198,12 @@ function createComputerToolSchema(actions: readonly ComputerUseV2ActionName[]) { windowRef: Type.Optional( Type.String({ description: "Opaque window reference from observation." }), ), + browserRef: Type.Optional( + Type.String({ description: "Opaque browser reference from get_browser_state." }), + ), + pageRef: Type.Optional( + Type.String({ description: "Opaque browser page reference from get_browser_state." }), + ), elementRef: Type.Optional( Type.String({ description: "Opaque accessibility element reference from observation." }), ), @@ -217,6 +230,30 @@ function createComputerToolSchema(actions: readonly ComputerUseV2ActionName[]) { "no_window_target", "other", ] as const), + snapshotFormat: optionalStringEnum(["dom_refs_v1", "semantic_v2"] as const), + continuation: Type.Optional(Type.String()), + includeScreenshot: Type.Optional(Type.Boolean()), + profile: optionalStringEnum(["isolated_new", "isolated_named"] as const), + profileName: Type.Optional(Type.String({ minLength: 1, maxLength: 64 })), + url: Type.Optional(Type.String()), + inputRoute: optionalStringEnum(["trusted", "dom_event"] as const), + mode: optionalStringEnum(["insert_text", "keystrokes"] as const), + replace: Type.Optional(Type.Boolean()), + dialogAction: optionalStringEnum(["inspect", "accept", "dismiss"] as const), + dialogRef: Type.Optional(Type.String()), + promptText: Type.Optional(Type.String()), + files: Type.Optional(Type.Array(Type.String({ minLength: 1 }), { minItems: 1, maxItems: 32 })), + destinationRoot: Type.Optional(Type.String()), + pointerAction: optionalStringEnum([ + "hover", + "right_click", + "double_click", + "scroll", + "drag", + ] as const), + destinationElementRef: Type.Optional(Type.String()), + deltaX: Type.Optional(Type.Number()), + deltaY: Type.Optional(Type.Number()), }); } @@ -298,6 +335,26 @@ function copyDeliveryMode(target: Record, input: Record, + input: Record, + key: string, +): void { + const value = input[key]; + if (value === undefined) { + return; + } + if (typeof value !== "boolean") { + throw new Error(`${key} must be a boolean`); + } + target[key] = value; +} + +function copyBrowserRefs(target: Record, input: Record): void { + target.browserRef = readToolStringParam(input, "browserRef", { required: true }); + target.pageRef = readToolStringParam(input, "pageRef", { required: true }); +} + /** Builds the computer.act wire params for one tool input action. */ function buildComputerActParams(params: { action: ComputerToolAction; @@ -434,6 +491,122 @@ function buildComputerActParams(params: { } break; } + case "get_browser_state": { + const windowRef = readToolStringParam(input, "windowRef"); + if (windowRef) { + wire.windowRef = windowRef; + break; + } + copyBrowserRefs(wire, input); + for (const key of [ + "snapshotFormat", + "elementRef", + "observationId", + "query", + "continuation", + ] as const) { + copyOptionalStringParam(wire, input, key); + } + copyOptionalBooleanParam(wire, input, "includeScreenshot"); + break; + } + case "browser_prepare": { + wire.windowRef = readToolStringParam(input, "windowRef", { required: true }); + copyOptionalStringParam(wire, input, "profile"); + copyOptionalStringParam(wire, input, "profileName"); + break; + } + case "browser_navigate": { + copyBrowserRefs(wire, input); + wire.url = readToolStringParam(input, "url", { required: true }); + break; + } + case "browser_click": { + copyBrowserRefs(wire, input); + wire.observationId = readToolStringParam(input, "observationId", { required: true }); + copyOptionalStringParam(wire, input, "elementRef"); + copyOptionalStringParam(wire, input, "inputRoute"); + const coordinate = readCoordinate(input, "coordinate"); + if (coordinate) { + wire.x = coordinate[0]; + wire.y = coordinate[1]; + } + break; + } + case "browser_type": { + copyBrowserRefs(wire, input); + for (const key of ["observationId", "elementRef"] as const) { + wire[key] = readToolStringParam(input, key, { required: true }); + } + wire.text = readToolStringParam(input, "text", { required: true, allowEmpty: true }); + copyOptionalStringParam(wire, input, "mode"); + copyOptionalBooleanParam(wire, input, "replace"); + break; + } + case "browser_dialog": { + copyBrowserRefs(wire, input); + wire.dialogAction = readToolStringParam(input, "dialogAction", { required: true }); + copyOptionalStringParam(wire, input, "dialogRef"); + copyOptionalStringParam(wire, input, "promptText"); + copyDeliveryMode(wire, input); + break; + } + case "browser_set_input_files": { + copyBrowserRefs(wire, input); + for (const key of ["observationId", "elementRef"] as const) { + wire[key] = readToolStringParam(input, key, { required: true }); + } + const files = input.files; + if ( + !Array.isArray(files) || + files.length < 1 || + files.length > 32 || + files.some((file) => typeof file !== "string" || !file) + ) { + throw new Error("files must contain 1-32 non-empty paths"); + } + wire.files = files; + break; + } + case "browser_download": { + copyBrowserRefs(wire, input); + for (const key of ["observationId", "elementRef", "destinationRoot"] as const) { + wire[key] = readToolStringParam(input, key, { required: true }); + } + break; + } + case "browser_pointer": { + copyBrowserRefs(wire, input); + wire.observationId = readToolStringParam(input, "observationId", { required: true }); + wire.pointerAction = readToolStringParam(input, "pointerAction", { required: true }); + for (const key of ["inputRoute", "elementRef", "destinationElementRef"] as const) { + copyOptionalStringParam(wire, input, key); + } + const coordinate = readCoordinate(input, "coordinate"); + if (coordinate) { + wire.x = coordinate[0]; + wire.y = coordinate[1]; + } + const destination = input.destinationCoordinate; + if (destination !== undefined) { + if ( + !Array.isArray(destination) || + destination.length !== 2 || + destination.some((value) => typeof value !== "number" || !Number.isFinite(value)) + ) { + throw new Error("destinationCoordinate must be a pair of finite numbers"); + } + wire.toX = destination[0]; + wire.toY = destination[1]; + } + for (const key of ["deltaX", "deltaY"] as const) { + const value = readFiniteNumberParam(input, key); + if (value !== undefined) { + wire[key] = value; + } + } + break; + } case "escalate_scope": { const reason = readToolStringParam(input, "reason", { required: true }); if (!ESCALATION_REASONS.has(reason)) { @@ -506,6 +679,7 @@ const READ_ONLY_COMPUTER_ACT_ACTIONS = new Set([ "get_cursor_position", "get_window_state", "zoom", + "get_browser_state", ]); function parseComputerActPayload(value: unknown): ComputerActResult { @@ -544,7 +718,22 @@ function computerActResultText(action: ComputerUseV2ActionName, result: Computer truncatedElements: observation.elements.length - MODEL_OBSERVATION_MAX_ELEMENTS, }; } - return JSON.stringify({ action, ...result, ...(observation ? { observation } : {}) }); + const details = result.details ? { ...result.details } : undefined; + if ( + details && + Array.isArray(details.elements) && + details.elements.length > MODEL_OBSERVATION_MAX_ELEMENTS + ) { + const originalLength = details.elements.length; + details.elements = details.elements.slice(0, MODEL_OBSERVATION_MAX_ELEMENTS); + details.truncatedElements = originalLength - MODEL_OBSERVATION_MAX_ELEMENTS; + } + return JSON.stringify({ + action, + ...result, + ...(observation ? { observation } : {}), + ...(details ? { details } : {}), + }); } async function invokeNodeCommand(params: { @@ -777,6 +966,8 @@ function validateCapabilityBoundInput(params: { }): void { const { capabilities, input } = params; const windowRef = readToolStringParam(input, "windowRef"); + const browserRef = readToolStringParam(input, "browserRef"); + const pageRef = readToolStringParam(input, "pageRef"); const elementRef = readToolStringParam(input, "elementRef"); const observationId = readToolStringParam(input, "observationId"); const deliveryMode = normalizeOptionalLowercaseString(input.deliveryMode); @@ -786,6 +977,9 @@ function validateCapabilityBoundInput(params: { if (elementRef && !capabilities?.targets.includes("element")) { throw new Error(`${COMPUTER_CONTRACT_MISMATCH}: selected node has no element target support`); } + if ((browserRef || pageRef) && !capabilities?.targets.includes("browser")) { + throw new Error(`${COMPUTER_CONTRACT_MISMATCH}: selected node has no browser target support`); + } if (deliveryMode && !capabilities?.deliveryModes.includes(deliveryMode as never)) { throw new Error( `${COMPUTER_CONTRACT_MISMATCH}: selected node does not advertise ${deliveryMode} delivery`, diff --git a/src/plugins/computer-use-contract.test.ts b/src/plugins/computer-use-contract.test.ts index e146753d4d21..8b430cc84fe4 100644 --- a/src/plugins/computer-use-contract.test.ts +++ b/src/plugins/computer-use-contract.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it, vi } from "vitest"; import { + COMPUTER_USE_CONTRACT_ONLY_ACTION_NAMES, COMPUTER_USE_V2_ACTION_NAMES, parseComputerActParamsJSON, parseComputerActResult, @@ -117,9 +118,41 @@ describe("Computer Use wire contract", () => { JSON.stringify({ action: "get_window_state", windowRef: "window-1", app: "wrong-family" }), ), ).toThrow("COMPUTER_INVALID_REQUEST"); - expect(() => parseComputerActParamsJSON(JSON.stringify({ action: "browser_click" }))).toThrow( - "COMPUTER_INVALID_REQUEST", - ); + expect( + parseComputerActParamsJSON( + JSON.stringify({ + action: "browser_click", + browserRef: "browser-1", + pageRef: "page-1", + observationId: "observation-1", + elementRef: "element-1", + inputRoute: "dom_event", + }), + ), + ).toMatchObject({ action: "browser_click", browserRef: "browser-1" }); + expect(() => + parseComputerActParamsJSON( + JSON.stringify({ + action: "browser_prepare", + windowRef: "window-1", + strategy: { kind: "existing_profile" }, + }), + ), + ).toThrow("COMPUTER_INVALID_REQUEST"); + }); + + it("keeps only the unimplemented recording family contract-gated", () => { + expect(COMPUTER_USE_CONTRACT_ONLY_ACTION_NAMES).toEqual([ + "get_recording_state", + "start_recording", + "stop_recording", + "replay_trajectory", + ]); + for (const action of COMPUTER_USE_CONTRACT_ONLY_ACTION_NAMES) { + expect(() => parseComputerActParamsJSON(JSON.stringify({ action }))).toThrow( + "COMPUTER_INVALID_REQUEST", + ); + } }); it("caps semantic observations and provider detail records", () => { diff --git a/src/plugins/computer-use-contract.ts b/src/plugins/computer-use-contract.ts index ab016c0514a9..b830f5a7fa6c 100644 --- a/src/plugins/computer-use-contract.ts +++ b/src/plugins/computer-use-contract.ts @@ -56,15 +56,6 @@ export const COMPUTER_USE_V1_ACTION_NAMES = COMPUTER_USE_V2_ACTION_NAMES.slice(0 export const COMPUTER_ACT_V1_ACTION_NAMES = COMPUTER_USE_V2_ACTION_NAMES.slice(1, 14); export const COMPUTER_USE_CONTRACT_ONLY_ACTION_NAMES = [ - "get_browser_state", - "browser_prepare", - "browser_navigate", - "browser_click", - "browser_type", - "browser_dialog", - "browser_set_input_files", - "browser_download", - "browser_pointer", "get_recording_state", "start_recording", "stop_recording", @@ -212,6 +203,104 @@ export const ComputerActParamsSchema = Type.Union([ x2: Type.Number({ minimum: 0 }), y2: Type.Number({ minimum: 0 }), }), + actionObject(["get_browser_state"], { + windowRef: Type.String({ minLength: 1 }), + }), + actionObject(["get_browser_state"], { + browserRef: Type.String({ minLength: 1 }), + pageRef: Type.String({ minLength: 1 }), + snapshotFormat: Type.Optional( + Type.Enum(["dom_refs_v1", "semantic_v2"] as const, { type: "string" }), + ), + elementRef: Type.Optional(Type.String({ minLength: 1 })), + observationId: Type.Optional(Type.String({ minLength: 1 })), + query: Type.Optional(Type.String()), + continuation: Type.Optional(Type.String({ minLength: 1 })), + includeScreenshot: Type.Optional(Type.Boolean()), + }), + actionObject(["browser_prepare"], { + windowRef: Type.String({ minLength: 1 }), + profile: Type.Optional( + Type.Enum(["isolated_new", "isolated_named"] as const, { type: "string" }), + ), + profileName: Type.Optional( + Type.String({ minLength: 1, maxLength: 64, pattern: "^[A-Za-z0-9._-]+$" }), + ), + }), + actionObject(["browser_navigate"], { + browserRef: Type.String({ minLength: 1 }), + pageRef: Type.String({ minLength: 1 }), + url: Type.String({ minLength: 1 }), + }), + actionObject(["browser_click"], { + browserRef: Type.String({ minLength: 1 }), + pageRef: Type.String({ minLength: 1 }), + observationId: Type.String({ minLength: 1 }), + elementRef: Type.Optional(Type.String({ minLength: 1 })), + x: Type.Optional(Type.Number({ minimum: 0 })), + y: Type.Optional(Type.Number({ minimum: 0 })), + inputRoute: Type.Optional(Type.Enum(["trusted", "dom_event"] as const, { type: "string" })), + }), + actionObject(["browser_type"], { + browserRef: Type.String({ minLength: 1 }), + pageRef: Type.String({ minLength: 1 }), + observationId: Type.String({ minLength: 1 }), + elementRef: Type.String({ minLength: 1 }), + text: Type.String(), + mode: Type.Optional(Type.Enum(["insert_text", "keystrokes"] as const, { type: "string" })), + replace: Type.Optional(Type.Boolean()), + }), + actionObject(["browser_dialog"], { + browserRef: Type.String({ minLength: 1 }), + pageRef: Type.String({ minLength: 1 }), + dialogAction: Type.Literal("inspect"), + }), + actionObject(["browser_dialog"], { + browserRef: Type.String({ minLength: 1 }), + pageRef: Type.String({ minLength: 1 }), + dialogAction: Type.Literal("accept"), + dialogRef: Type.String({ minLength: 1 }), + promptText: Type.Optional(Type.String()), + deliveryMode: Type.Optional(Type.Enum(DELIVERY_MODES, { type: "string" })), + }), + actionObject(["browser_dialog"], { + browserRef: Type.String({ minLength: 1 }), + pageRef: Type.String({ minLength: 1 }), + dialogAction: Type.Literal("dismiss"), + dialogRef: Type.String({ minLength: 1 }), + deliveryMode: Type.Optional(Type.Enum(DELIVERY_MODES, { type: "string" })), + }), + actionObject(["browser_set_input_files"], { + browserRef: Type.String({ minLength: 1 }), + pageRef: Type.String({ minLength: 1 }), + observationId: Type.String({ minLength: 1 }), + elementRef: Type.String({ minLength: 1 }), + files: Type.Array(Type.String({ minLength: 1 }), { minItems: 1, maxItems: 32 }), + }), + actionObject(["browser_download"], { + browserRef: Type.String({ minLength: 1 }), + pageRef: Type.String({ minLength: 1 }), + observationId: Type.String({ minLength: 1 }), + elementRef: Type.String({ minLength: 1 }), + destinationRoot: Type.String({ minLength: 1 }), + }), + actionObject(["browser_pointer"], { + browserRef: Type.String({ minLength: 1 }), + pageRef: Type.String({ minLength: 1 }), + observationId: Type.String({ minLength: 1 }), + pointerAction: Type.Enum(["hover", "right_click", "double_click", "scroll", "drag"] as const, { + type: "string", + }), + inputRoute: Type.Optional(Type.Enum(["trusted", "dom_event"] as const, { type: "string" })), + elementRef: Type.Optional(Type.String({ minLength: 1 })), + x: Type.Optional(Type.Number({ minimum: 0 })), + y: Type.Optional(Type.Number({ minimum: 0 })), + destinationElementRef: Type.Optional(Type.String({ minLength: 1 })), + toX: Type.Optional(Type.Number({ minimum: 0 })), + toY: Type.Optional(Type.Number({ minimum: 0 })), + deltaX: Type.Optional(Type.Number()), + deltaY: Type.Optional(Type.Number()), + }), actionObject(["escalate_scope"], { reason: Type.Enum(ESCALATION_REASONS, { type: "string" }), }), @@ -233,7 +322,7 @@ const ComputerBoundsSchema = Type.Object( const ComputerObservationSchema = Type.Object( { - kind: Type.Enum(["window", "screen"] as const, { type: "string" }), + kind: Type.Enum(["window", "screen", "browser"] as const, { type: "string" }), base64: Type.Optional(Type.String()), format: Type.Optional(Type.Enum(["jpeg", "png"] as const, { type: "string" })), width: Type.Optional(Type.Integer({ minimum: 1 })),