mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-27 12:56:01 -06:00
test(computer-use): live end-to-end gate for both macOS providers (#123991)
* fix(computer-use): preserve semantic click observations * fix(cua-computer): isolate desktop and window sessions * fix(cua-computer): route cursor reads through desktop scope * test(computer-use): add isolated macOS live proof rig * style(cua-computer): format driver sessions * fix(cua-computer): forward lazy desktop tools * fix(computer-use): narrow live proof artifact path * test(computer-use): split schema contract coverage * refactor(cua-computer): unify session start state * build(computer-use): register live proof entrypoint * style(computer-use): satisfy full lint contract * fix(computer-use): harden live proof authority * fix(cua-computer): keep window authority immutable * fix(computer-use): close partial sessions safely
This commit is contained in:
Peter Steinberger
committed by
GitHub
GitHub
parent
2adbdd75c6
commit
a8f1a0d345
@@ -70,6 +70,7 @@ final class ComputerActionServiceV2 {
|
||||
private let windows: WindowManagementService
|
||||
private let menu: MenuService
|
||||
private let observationService: DesktopObservationService
|
||||
private let snapshotManager: InMemorySnapshotManager
|
||||
private var lifecycleGeneration: UInt64?
|
||||
private var appRefs: [String: ServiceApplicationInfo] = [:]
|
||||
private var windowRefs: [String: WindowTarget] = [:]
|
||||
@@ -77,7 +78,7 @@ final class ComputerActionServiceV2 {
|
||||
private var executionAuthority: ComputerActionExecutionAuthority?
|
||||
|
||||
init() {
|
||||
let snapshotManager = SnapshotManager()
|
||||
let snapshotManager = InMemorySnapshotManager()
|
||||
let automation = UIAutomationService(snapshotManager: snapshotManager)
|
||||
let applications = ApplicationService()
|
||||
let menu = MenuService(applicationService: applications)
|
||||
@@ -85,6 +86,7 @@ final class ComputerActionServiceV2 {
|
||||
self.applications = applications
|
||||
self.windows = WindowManagementService(applicationService: applications)
|
||||
self.menu = menu
|
||||
self.snapshotManager = snapshotManager
|
||||
self.observationService = DesktopObservationService(
|
||||
screenCapture: ScreenCaptureService(loggingService: LoggingService()),
|
||||
automation: automation,
|
||||
@@ -281,6 +283,13 @@ final class ComputerActionServiceV2 {
|
||||
let result = try await self.withExecutionAuthority {
|
||||
try await self.observationService.observe(request)
|
||||
}
|
||||
if let elements = result.elements {
|
||||
try await self.withExecutionAuthority {
|
||||
try await self.snapshotManager.storeDetectionResult(
|
||||
snapshotId: elements.snapshotId,
|
||||
result: elements)
|
||||
}
|
||||
}
|
||||
let observedWindow = result.capture.metadata.windowInfo ?? target.window
|
||||
guard observedWindow.windowID == target.window.windowID else {
|
||||
throw ComputerActionService.ComputerActionError.staleObservation
|
||||
|
||||
@@ -24,6 +24,8 @@ const repositoryScriptEntries = [
|
||||
"scripts/check-package-dist-imports.mjs!",
|
||||
// Cloudflare deployment template: wrangler bundles the Worker from this entry.
|
||||
"scripts/cloudflare/src/index.ts!",
|
||||
// Invoked by the documented macOS Computer Use live-proof shell rig.
|
||||
"scripts/dev/computer-use-macos-live-proof.ts!",
|
||||
"scripts/dev/ios-node-e2e.ts!",
|
||||
"scripts/diffs-shiki-curated.ts!",
|
||||
// Reusable Docker workflows invoke this from the downloaded .release-harness tree.
|
||||
|
||||
@@ -62,6 +62,31 @@ The CUA descriptor advertises window, element, and browser targets; background a
|
||||
|
||||
Browser targets, pages, page elements, and dialogs are opaque capabilities. Retake browser state after navigation, reconnect, or a stale-reference refusal. The adapter never returns provider-native CDP target IDs, tab IDs, or page refs to the model.
|
||||
|
||||
### Maintainer live-proof rig
|
||||
|
||||
The repository includes a macOS-only development rig that preserves the real vertical path: agent-facing `computer` tool, Gateway `node.invoke`, paired Mac node, and the selected node-local provider. It is deliberately isolated from the operator app and Gateway.
|
||||
|
||||
Build a signed app from a clean, committed checkout, choose a fresh profile and non-default loopback port, and prepare the two config views:
|
||||
|
||||
```bash
|
||||
scratch="$(mktemp -d /tmp/openclaw-cu-live.XXXXXX)"
|
||||
scripts/dev/computer-use-macos-live-rig.sh prepare \
|
||||
cu-live-proof 29431 "$PWD/dist/OpenClaw.app" "$scratch" peekaboo
|
||||
```
|
||||
|
||||
Run the emitted `gateway` and `app` commands in separate terminals. The split config is intentional: the externally launched daemon reads a scratch config with `gateway.mode: "local"`, while the app profile reads `gateway.mode: "remote"`, direct transport, and the daemon's loopback URL. If the app reads local mode, its Port Guardian owns the route instead of joining the external daemon. The rig keeps its validated launch fields in non-executable `rig.json`; later commands reject unknown fields or paths that do not match the scratch/profile layout. It also seeds a dedicated `node` identity, completed onboarding, unpaused state, Computer Control, and the checkout path used to start the debug node worker. There is no separate node-mode toggle.
|
||||
|
||||
In a third terminal, run the emitted `nodes` command. A fresh CLI identity first returns a device-approval request; approve that request from the isolated app's Devices settings or with `openclaw --profile cu-live-proof devices approve <requestId>`, then rerun `nodes` until the paired entry is connected and advertises `computer.act` plus a `computerUse` descriptor.
|
||||
|
||||
Place a harmless editable fixture window behind a different frontmost app, then run the vertical:
|
||||
|
||||
```bash
|
||||
scripts/dev/computer-use-macos-live-rig.sh proof \
|
||||
"$scratch" peekaboo "Computer Use Fixture" "background proof" "Editor"
|
||||
```
|
||||
|
||||
The proof runner first requires the sole connected computer node to advertise the requested provider, then executes `screenshot`, `list_windows`, `get_window_state`, background element click and type, and re-observes the window. It saves the structured result and target-window before/after images under the scratch directory and fails unless the provider matches, the target started non-frontmost, the frontmost app and cursor stayed unchanged, target content changed, and the final effect was confirmed or a structured refusal. Restart the isolated app with the other provider and rerun the same proof. Do not use port `18789`, the default profile, or `/Applications/OpenClaw.app` for this rig.
|
||||
|
||||
### Windows and Linux (experimental, direct SDK)
|
||||
|
||||
The bundled `cua-computer` plugin provides an experimental fulfiller for Windows and Linux node hosts. It is disabled by default and uses the pinned CUA Driver SDK 0.19.3 contract directly:
|
||||
@@ -80,7 +105,7 @@ The bundled `cua-computer` plugin provides an experimental fulfiller for Windows
|
||||
|
||||
OpenClaw checks the SDK package version, the selected OS/CPU package version, regular-file identity, and the pinned SHA-256 digest of the native library and Node runtime. A clean check prints `no findings`. If it reports a `COMPUTER_DRIVER_*` error, reinstall or update OpenClaw on this node host and run the check again. Do not download a standalone `cua-driver` executable or add one to `PATH`; Windows and Linux use the npm-installed in-process SDK.
|
||||
|
||||
3. Start `openclaw node run` from the interactive desktop session. The plugin repeats the artifact verification at startup before it imports native code, creates its configured SDK runtime lazily, then creates one OpenClaw-owned trusted session for the node-host command execution. It closes that session and shuts down the runtime when the command host stops or restarts.
|
||||
3. Start `openclaw node run` from the interactive desktop session. The plugin repeats the artifact verification at startup before it imports native code, creates its configured SDK runtime lazily, then creates separate fixed window- and desktop-scoped trusted sessions for node-host command execution. `escalate_scope` reads the already-desktop session state, so the window identity remains immutable. It closes both sessions and shuts down the runtime when the command host stops or restarts.
|
||||
|
||||
4. Add `computer.act` to the Gateway allowlist. This plugin registers `computer.act` as a dangerous plugin node command, so enabling the plugin alone is not enough; the operator must opt in explicitly:
|
||||
|
||||
@@ -96,7 +121,7 @@ The bundled `cua-computer` plugin provides an experimental fulfiller for Windows
|
||||
|
||||
This fulfiller currently controls only the primary display. `hold_key`, `left_mouse_down`, and `left_mouse_up` are unavailable because the CUA Driver SDK has no desktop-scope held-input contract. Modifier-held clicks, scrolling, and dragging are rejected because the typed desktop methods do not accept modifiers. The `key` action accepts named keys, letters, and modifier combos (for example `cmd+c` or `Return`); digit and punctuation keys are rejected because the driver drops their layout-dependent shift state, so send that text through the `type` action instead. Cancellation is passed to the SDK for each node invocation.
|
||||
|
||||
The plugin calls `CuaDriver.createConfigured`, never bare `create()`. Its authorization ceiling, trusted session identifier, TTLs, and desktop scope are fixed by OpenClaw; model-facing `screen.snapshot` and `computer.act` inputs cannot select a session or widen that authority. Because the driver reports no stable display identity, frame authorization binds to the trusted session generation plus live primary-display geometry. A new session invalidates outstanding frames, but a same-geometry primary-display substitution inside one session cannot be detected; prefer a stable single-display session for this fulfiller.
|
||||
The plugin calls `CuaDriver.createConfigured`, never bare `create()`. Its authorization ceiling, fixed window/desktop session identities, TTLs, and scopes are owned by OpenClaw; model-facing `screen.snapshot` and `computer.act` inputs cannot select a session or widen the window identity. Because the driver reports no stable display identity, frame authorization binds to the trusted session generation plus live primary-display geometry. A new session invalidates outstanding frames, but a same-geometry primary-display substitution inside one session cannot be detected; prefer a stable single-display session for this fulfiller.
|
||||
|
||||
On Windows and Linux this is a hard replacement of the former 0.10 daemon/MCP integration: OpenClaw does not spawn a CUA process or proxy an MCP client. macOS deliberately uses the app-owned embedded daemon described above so the driver remains in `OpenClaw.app`'s TCC responsibility chain. Neither path falls back to another provider for an individual action.
|
||||
|
||||
|
||||
+27
-20
@@ -124,26 +124,27 @@ Provider capability truth (verified in source, 2026-08-13):
|
||||
Waves follow RFC 0025's implementation plan, compressed by the no-compat ruling.
|
||||
Status: `todo | in-progress | pr | landed | blocked`.
|
||||
|
||||
| ID | Work | Repo | Status | Notes |
|
||||
| ---------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- | ------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| W0-FIX | Parity matrix + pinned fixtures: 49 CUA tools + 25 Peekaboo tools classified against the v2 union; recorded tools/list + result fixtures | openclaw | landed | landed #123469 (d9646ad): 58 CUA + 26 Peekaboo tools, +1059 test-only LOC |
|
||||
| W0-PIN | Pick + pin CUA release with embedded/inherited-IPC support; dependency bump (needs Dependency Guard approval) | openclaw | landed | resolved: 0.19.3 already contains #2410+#2411 |
|
||||
| W1-PROTO | v2 types: action union (+`invoke_menu`), target union, capability descriptor, result envelope, closed error codes; **replaces** v1 params in place | openclaw | landed | landed #123544 (d19c755) |
|
||||
| W1-TOOL | Built-in computer tool v2: capability-filtered schema, observation refs, result projection, no auto-retry | openclaw | landed | landed #123544: v1 651 / v2 742 tokens |
|
||||
| W1-SEAM | Node-host provider seam: one registration, provider selection, generation, lifecycle close path; absorbs cua-computer's direct registration | openclaw | landed | landed #123509 (848a7e3): seam+contract, prod +348/-217 |
|
||||
| W2-CUA | CUA plugin refactor onto seam: full v2 mapping (window/element/background), session-per-execution, deletes desktop-scope-only adapter | openclaw | landed | landed #123604 (2af5eca): full v2 adapter, prod +1129/-41 |
|
||||
| W2-MAC | macOS app: bundle + re-sign driver, direct `serve --embedded` spawn, private socket handoff to worker, TCC restart handling | openclaw | landed | landed #123635 (19ace68): app-owned daemon + picker + orphan reaping, live-proven |
|
||||
| W2-PKB | Peekaboo adapter on the same seam (macOS): see/click/type/press/set_value/verify_state/app/window/menu mapping | openclaw | landed | landed #123801 (4a6edc0): native v2, live proof deferred to W3-GATE |
|
||||
| W2-PKB-UP | Peekaboo upstream: middle/triple click, hold_key + mouse down/up in BackgroundInputDriver, get_cursor_position; optional browser-shape alignment | Peekaboo | todo | owner-approved |
|
||||
| W2-UX | Settings -> Computer Use provider picker + readiness checklist (both apps: macOS now, Tauri Linux later) | openclaw | todo | RFC OC-10B slice; owns picker screenshots (W2-MAC shipped without them; app instance lock at `/tmp/openclaw-UID-app-instances` can block a fresh profile launch) |
|
||||
| W3-GATE | Integration gate: live vertical on macOS (both providers) + Linux X11 (CUA): observe window -> background element click -> verify | openclaw | todo | RFC OC-8 |
|
||||
| W3-SKILL | Version-pinned skill profile: background-first ladder, result precedence, no CLI/daemon instructions | openclaw | todo | RFC OC-9D |
|
||||
| W4-BROWSER | CUA browser family (isolated profile first; existing-profile gated on consent adapter) | openclaw | todo | RFC OC-9B; optional family |
|
||||
| W4-REC | CUA recording/resources family with node-owned roots | openclaw | todo | RFC OC-9C; optional family |
|
||||
| W4-WIN | Windows companion CUA host PR | openclaw-windows-node | todo | RFC WIN-1; after W1-SEAM/W2-CUA |
|
||||
| W4-ART | Managed artifacts: Win/Linux digest-pinned download, atomic update + rollback | openclaw | todo | RFC OC-10A |
|
||||
| W5-SEC | Security closure: high-risk action classification, socket ownership audit, hostile-arg tests | openclaw | todo | RFC OC-10C |
|
||||
| W5-ACC | Packaged cross-platform acceptance + default-provider rollout | openclaw | todo | RFC OC-11/12 |
|
||||
| ID | Work | Repo | Status | Notes |
|
||||
| ---------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- | ------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| W0-FIX | Parity matrix + pinned fixtures: 49 CUA tools + 25 Peekaboo tools classified against the v2 union; recorded tools/list + result fixtures | openclaw | landed | landed #123469 (d9646ad): 58 CUA + 26 Peekaboo tools, +1059 test-only LOC |
|
||||
| W0-PIN | Pick + pin CUA release with embedded/inherited-IPC support; dependency bump (needs Dependency Guard approval) | openclaw | landed | resolved: 0.19.3 already contains #2410+#2411 |
|
||||
| W1-PROTO | v2 types: action union (+`invoke_menu`), target union, capability descriptor, result envelope, closed error codes; **replaces** v1 params in place | openclaw | landed | landed #123544 (d19c755) |
|
||||
| W1-TOOL | Built-in computer tool v2: capability-filtered schema, observation refs, result projection, no auto-retry | openclaw | landed | landed #123544: v1 651 / v2 742 tokens |
|
||||
| W1-SEAM | Node-host provider seam: one registration, provider selection, generation, lifecycle close path; absorbs cua-computer's direct registration | openclaw | landed | landed #123509 (848a7e3): seam+contract, prod +348/-217 |
|
||||
| W2-CUA | CUA plugin refactor onto seam: full v2 mapping (window/element/background), session-per-execution, deletes desktop-scope-only adapter | openclaw | landed | landed #123604 (2af5eca): full v2 adapter, prod +1129/-41 |
|
||||
| W2-MAC | macOS app: bundle + re-sign driver, direct `serve --embedded` spawn, private socket handoff to worker, TCC restart handling | openclaw | landed | landed #123635 (19ace68): app-owned daemon + picker + orphan reaping, live-proven |
|
||||
| W2-PKB | Peekaboo adapter on the same seam (macOS): see/click/type/press/set_value/verify_state/app/window/menu mapping | openclaw | landed | landed #123801 (4a6edc0): native v2; live vertical completed by W3-GATE |
|
||||
| W2-PKB-UP | Peekaboo upstream: middle/triple click, hold_key + mouse down/up in BackgroundInputDriver, get_cursor_position; optional browser-shape alignment | Peekaboo | todo | owner-approved |
|
||||
| W2-UX | Settings -> Computer Use provider picker + readiness checklist (both apps: macOS now, Tauri Linux later) | openclaw | todo | RFC OC-10B slice; owns picker screenshots (W2-MAC shipped without them; app instance lock at `/tmp/openclaw-UID-app-instances` can block a fresh profile launch) |
|
||||
| W3-GATE | Integration gate: live vertical on macOS (both providers): observe window -> background element click/type -> verify | openclaw | landed | live agent-tool -> Gateway -> Mac-node proof: both providers kept frontmost app/cursor unchanged and changed target content; CUA confirmed type readback, Peekaboo confirmed `set_value` after honest unverifiable click/type evidence |
|
||||
| W3-LINUX | Integration gate: live vertical on Linux X11 (CUA): observe window -> background element click -> verify | openclaw | todo | split from W3-GATE; no Linux proof claimed by the macOS gate |
|
||||
| W3-SKILL | Version-pinned skill profile: background-first ladder, result precedence, no CLI/daemon instructions | openclaw | todo | RFC OC-9D |
|
||||
| W4-BROWSER | CUA browser family (isolated profile first; existing-profile gated on consent adapter) | openclaw | todo | RFC OC-9B; optional family |
|
||||
| W4-REC | CUA recording/resources family with node-owned roots | openclaw | todo | RFC OC-9C; optional family |
|
||||
| W4-WIN | Windows companion CUA host PR | openclaw-windows-node | todo | RFC WIN-1; after W1-SEAM/W2-CUA |
|
||||
| W4-ART | Managed artifacts: Win/Linux digest-pinned download, atomic update + rollback | openclaw | todo | RFC OC-10A |
|
||||
| W5-SEC | Security closure: high-risk action classification, socket ownership audit, hostile-arg tests | openclaw | todo | RFC OC-10C |
|
||||
| W5-ACC | Packaged cross-platform acceptance + default-provider rollout | openclaw | todo | RFC OC-11/12 |
|
||||
|
||||
Production LOC ledger (updated per landed PR): net target ≤ +1500 for the whole
|
||||
campaign excluding tests/fixtures, funded by deleting the v1-only branches, the
|
||||
@@ -159,6 +160,12 @@ Every wave carries live proof; unit fixtures alone never advance the tracker.
|
||||
keeps focus; assert frontmost app unchanged, cursor position unchanged,
|
||||
`verify_state`/`effect` confirms the edit. Video via screen recording for
|
||||
UX-visible changes.
|
||||
- W3-GATE executed on 2026-08-14 through the real agent tool and paired-node
|
||||
route. Peekaboo preserved Finder as frontmost and its cursor position. CUA
|
||||
preserved VLC and its cursor position. Both changed the background target.
|
||||
CUA returned confirmed accessibility value readback; Peekaboo honestly marked
|
||||
click/type delivery unverifiable and confirmed the follow-up background
|
||||
`set_value` with verified change evidence.
|
||||
- **Linux CUA**: Crabbox Xvfb host (recipe proven in PR #117205): node registers
|
||||
command pair, real screenshot + frame id, background window action against
|
||||
xterm/gtk test app; Wayland smoke on Sway only.
|
||||
|
||||
@@ -86,6 +86,7 @@ export function driver(
|
||||
const typeText = vi.fn(async () => result({}));
|
||||
const pressKey = vi.fn(async () => result({}));
|
||||
const callTool = vi.fn<CuaDriverSession["callTool"]>(async () => result({}));
|
||||
const callDesktopTool = vi.fn<CuaDriverSession["callDesktopTool"]>(async () => result({}));
|
||||
const escalateScope = vi.fn(async () => ({
|
||||
session: "openclaw-test",
|
||||
captureScope: 2,
|
||||
@@ -100,6 +101,7 @@ export function driver(
|
||||
isAvailable: () => true,
|
||||
resetAvailabilityCache: () => {},
|
||||
callTool,
|
||||
callDesktopTool,
|
||||
escalateScope,
|
||||
getDesktopState,
|
||||
getScreenSize,
|
||||
@@ -120,6 +122,7 @@ export function driver(
|
||||
moveCursor,
|
||||
scroll,
|
||||
callTool,
|
||||
callDesktopTool,
|
||||
escalateScope,
|
||||
dispose,
|
||||
typeText,
|
||||
|
||||
@@ -552,7 +552,7 @@ describe("cua-computer provider", () => {
|
||||
});
|
||||
|
||||
it("maps remaining discovery, window lifecycle, and semantic actions", async () => {
|
||||
const { session, callTool } = driver();
|
||||
const { session, callTool, callDesktopTool } = driver();
|
||||
callTool.mockImplementation(async (name) => {
|
||||
if (name === "list_windows") {
|
||||
return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.listWindows);
|
||||
@@ -566,9 +566,6 @@ describe("cua-computer provider", () => {
|
||||
windows: CUA_DRIVER_CONTRACT_FIXTURES.listWindows.windows,
|
||||
});
|
||||
}
|
||||
if (name === "get_cursor_position") {
|
||||
return cuaToolResult({ x: 11, y: 12, source: "x11" });
|
||||
}
|
||||
return cuaToolResult(
|
||||
{},
|
||||
{
|
||||
@@ -577,6 +574,11 @@ describe("cua-computer provider", () => {
|
||||
},
|
||||
);
|
||||
});
|
||||
callDesktopTool.mockImplementation(async (name) =>
|
||||
name === "get_cursor_position"
|
||||
? cuaToolResult({ x: 11, y: 12, source: "x11" })
|
||||
: cuaToolResult({}),
|
||||
);
|
||||
const computer = await execution(session);
|
||||
const tree = JSON.parse(await computer.act('{"action":"get_accessibility_tree"}')) as {
|
||||
details: { windows: unknown[]; processes: unknown[] };
|
||||
@@ -584,6 +586,7 @@ describe("cua-computer provider", () => {
|
||||
expect(tree.details.windows).toHaveLength(1);
|
||||
expect(tree.details.processes).toHaveLength(1);
|
||||
await expect(computer.act('{"action":"get_cursor_position"}')).resolves.toContain('"x":11');
|
||||
expect(callDesktopTool).toHaveBeenCalledWith("get_cursor_position", {}, undefined);
|
||||
|
||||
const listed = JSON.parse(await computer.act('{"action":"list_windows"}')) as {
|
||||
details: { windows: Array<{ windowRef: string }> };
|
||||
|
||||
@@ -448,7 +448,12 @@ export const COMPUTER_USE_V2_PROVIDER_ACTION_SUPPORT: readonly ComputerUseV2Prov
|
||||
cuaTools: ["browser_pointer"],
|
||||
peekabooTools: ["browser"],
|
||||
},
|
||||
{ action: "escalate_scope", support: "cua", cuaTools: ["escalate_session"], peekabooTools: [] },
|
||||
{
|
||||
action: "escalate_scope",
|
||||
support: "cua",
|
||||
cuaTools: ["get_session_state"],
|
||||
peekabooTools: [],
|
||||
},
|
||||
{
|
||||
action: "get_recording_state",
|
||||
support: "cua",
|
||||
|
||||
@@ -461,14 +461,15 @@ export const CUA_MCP_TOOL_PARITY: readonly CuaMcpToolClassification[] = [
|
||||
{
|
||||
tool: "escalate_session",
|
||||
platforms: CUA_ALL_PLATFORMS,
|
||||
classification: "portable-action",
|
||||
actions: ["escalate_scope"],
|
||||
classification: "node-internal-lifecycle",
|
||||
reason:
|
||||
"The fixed desktop session already satisfies escalation without mutating window authority.",
|
||||
},
|
||||
{
|
||||
tool: "get_session_state",
|
||||
platforms: CUA_ALL_PLATFORMS,
|
||||
classification: "node-internal-lifecycle",
|
||||
reason: "Provider session state is owned by the node execution.",
|
||||
classification: "portable-action",
|
||||
actions: ["escalate_scope"],
|
||||
},
|
||||
{
|
||||
tool: "end_session",
|
||||
|
||||
@@ -14,6 +14,12 @@ const mocks = vi.hoisted(() => ({
|
||||
desktopUnlocked: true,
|
||||
})),
|
||||
getDesktopState: vi.fn(async () => ({})),
|
||||
getSessionState: vi.fn(async () => ({
|
||||
session: "openclaw-test",
|
||||
captureScope: "desktop",
|
||||
effectiveScope: "desktop",
|
||||
desktopUnlocked: true,
|
||||
})),
|
||||
isAvailable: vi.fn(() => true),
|
||||
startSession: vi.fn(async () => ({})),
|
||||
shutdown: vi.fn(async () => {}),
|
||||
@@ -31,7 +37,12 @@ const sdk = {
|
||||
createTrustedSession: mocks.createTrustedSession,
|
||||
};
|
||||
|
||||
import { ClickButton, createCuaDriver, ScrollDirection } from "./driver-client.js";
|
||||
import {
|
||||
ClickButton,
|
||||
createCuaDriver,
|
||||
EscalationReason,
|
||||
ScrollDirection,
|
||||
} from "./driver-client.js";
|
||||
|
||||
const authorization = {
|
||||
allowedModes: ["unrestricted"],
|
||||
@@ -54,6 +65,7 @@ describe("CUA Driver direct session", () => {
|
||||
endSession: mocks.endSession,
|
||||
escalateSession: mocks.escalateSession,
|
||||
getDesktopState: mocks.getDesktopState,
|
||||
getSessionState: mocks.getSessionState,
|
||||
startSession: mocks.startSession,
|
||||
});
|
||||
});
|
||||
@@ -74,7 +86,7 @@ describe("CUA Driver direct session", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("uses configured creation and one fixed trusted OpenClaw session", async () => {
|
||||
it("uses configured creation and fixed window and desktop sessions", async () => {
|
||||
const driver = createCuaDriver({ loadSdk: () => sdk as never });
|
||||
|
||||
expect(driver.isAvailable()).toBe(true);
|
||||
@@ -83,19 +95,21 @@ describe("CUA Driver direct session", () => {
|
||||
claudeCodeCompatibility: false,
|
||||
authorization,
|
||||
});
|
||||
expect(mocks.createTrustedSession).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
publicSession: expect.stringMatching(/^openclaw-/),
|
||||
mode: "unrestricted",
|
||||
ttlSeconds: authorization.maxSessionTtlSeconds,
|
||||
idleTtlSeconds: authorization.maxIdleTtlSeconds,
|
||||
}),
|
||||
);
|
||||
expect(mocks.createTrustedSession).toHaveBeenCalledTimes(2);
|
||||
for (const [, options] of mocks.createTrustedSession.mock.calls) {
|
||||
expect(options).toEqual(
|
||||
expect.objectContaining({
|
||||
publicSession: expect.stringMatching(/^openclaw-(window|desktop)-/),
|
||||
mode: "unrestricted",
|
||||
ttlSeconds: authorization.maxSessionTtlSeconds,
|
||||
idleTtlSeconds: authorization.maxIdleTtlSeconds,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
await driver.dispose();
|
||||
await driver.dispose();
|
||||
expect(mocks.close).toHaveBeenCalledOnce();
|
||||
expect(mocks.close).toHaveBeenCalledTimes(2);
|
||||
expect(mocks.shutdown).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
@@ -103,7 +117,9 @@ describe("CUA Driver direct session", () => {
|
||||
const driver = createCuaDriver({ loadSdk: () => sdk as never });
|
||||
|
||||
await Promise.all([driver.getDesktopState(), driver.getDesktopState()]);
|
||||
const sessionOptions = mocks.createTrustedSession.mock.calls[0]?.[1];
|
||||
const sessionOptions = mocks.createTrustedSession.mock.calls.find(([, options]) =>
|
||||
options.publicSession.startsWith("openclaw-desktop-"),
|
||||
)?.[1];
|
||||
|
||||
expect(mocks.startSession).toHaveBeenCalledOnce();
|
||||
expect(mocks.startSession).toHaveBeenCalledWith(
|
||||
@@ -119,28 +135,49 @@ describe("CUA Driver direct session", () => {
|
||||
expect(mocks.endSession).toHaveBeenCalledWith({ session: sessionOptions.publicSession });
|
||||
});
|
||||
|
||||
it("starts window-scoped generic tools and widens only for an explicit desktop call", async () => {
|
||||
it("keeps the window session immutable across desktop actions and escalation", async () => {
|
||||
const driver = createCuaDriver({ loadSdk: () => sdk as never });
|
||||
|
||||
await driver.getDesktopState();
|
||||
await driver.callTool("list_windows", {});
|
||||
const sessionOptions = mocks.createTrustedSession.mock.calls[0]?.[1];
|
||||
await driver.callDesktopTool("get_cursor_position", {});
|
||||
await driver.escalateScope(EscalationReason.Other);
|
||||
await driver.callTool("list_windows", {});
|
||||
const windowOptions = mocks.createTrustedSession.mock.calls.find(([, options]) =>
|
||||
options.publicSession.startsWith("openclaw-window-"),
|
||||
)?.[1];
|
||||
const desktopOptions = mocks.createTrustedSession.mock.calls.find(([, options]) =>
|
||||
options.publicSession.startsWith("openclaw-desktop-"),
|
||||
)?.[1];
|
||||
expect(mocks.startSession).toHaveBeenCalledWith(
|
||||
{ session: sessionOptions.publicSession, captureScope: "window" },
|
||||
{ session: windowOptions.publicSession, captureScope: "window" },
|
||||
undefined,
|
||||
);
|
||||
expect(mocks.startSession).toHaveBeenCalledWith(
|
||||
{ session: desktopOptions.publicSession, captureScope: "desktop" },
|
||||
undefined,
|
||||
);
|
||||
expect(mocks.callTool).toHaveBeenCalledTimes(3);
|
||||
expect(mocks.callTool).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
"list_windows",
|
||||
JSON.stringify({ session: windowOptions.publicSession }),
|
||||
undefined,
|
||||
);
|
||||
expect(mocks.callTool).toHaveBeenCalledWith(
|
||||
"list_windows",
|
||||
JSON.stringify({ session: sessionOptions.publicSession }),
|
||||
"get_cursor_position",
|
||||
JSON.stringify({ session: desktopOptions.publicSession }),
|
||||
undefined,
|
||||
);
|
||||
|
||||
await driver.getDesktopState();
|
||||
expect(mocks.escalateSession).toHaveBeenCalledWith(
|
||||
{
|
||||
session: sessionOptions.publicSession,
|
||||
reason: "other",
|
||||
detail: "explicit desktop-scope OpenClaw action",
|
||||
},
|
||||
expect(mocks.getSessionState).toHaveBeenCalledWith(
|
||||
{ session: desktopOptions.publicSession },
|
||||
undefined,
|
||||
);
|
||||
expect(mocks.escalateSession).not.toHaveBeenCalled();
|
||||
expect(mocks.callTool).toHaveBeenNthCalledWith(
|
||||
3,
|
||||
"list_windows",
|
||||
JSON.stringify({ session: windowOptions.publicSession }),
|
||||
undefined,
|
||||
);
|
||||
await driver.dispose();
|
||||
|
||||
@@ -2,7 +2,6 @@ import { randomUUID } from "node:crypto";
|
||||
import { verifyInstalledCuaDriverArtifacts } from "./driver-artifacts.js";
|
||||
|
||||
type DriverClickButton = import("@trycua/cua-driver").ClickButton;
|
||||
type DriverCaptureScope = import("@trycua/cua-driver").CaptureScope;
|
||||
type DriverEscalationReason = import("@trycua/cua-driver").EscalationReason;
|
||||
type CuaDriverLike = import("@trycua/cua-driver").CuaDriverLike;
|
||||
type CuaDriverSessionLike = import("@trycua/cua-driver").CuaDriverSessionLike;
|
||||
@@ -57,6 +56,11 @@ export interface CuaDriverSession {
|
||||
args: Record<string, unknown>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<CuaToolResult>;
|
||||
callDesktopTool(
|
||||
name: string,
|
||||
args: Record<string, unknown>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<CuaToolResult>;
|
||||
escalateScope(reason: EscalationReason, signal?: AbortSignal): Promise<CuaSessionState>;
|
||||
getDesktopState(signal?: AbortSignal): Promise<CuaToolResult>;
|
||||
getScreenSize(signal?: AbortSignal): Promise<CuaToolResult>;
|
||||
@@ -88,12 +92,14 @@ function asyncOptions(signal?: AbortSignal) {
|
||||
class DirectCuaDriverSession implements CuaDriverSession {
|
||||
readonly generation = randomUUID();
|
||||
private readonly runtime: CuaDriverLike;
|
||||
private readonly session: CuaDriverSessionLike;
|
||||
private readonly publicSession = `openclaw-${randomUUID()}`;
|
||||
private startPromise: Promise<void> | undefined;
|
||||
private desktopEscalationPromise: Promise<void> | undefined;
|
||||
private captureScope: DriverCaptureScope | undefined;
|
||||
private started = false;
|
||||
private readonly windowSession: CuaDriverSessionLike;
|
||||
private readonly desktopSession: CuaDriverSessionLike;
|
||||
private readonly windowPublicSession = `openclaw-window-${randomUUID()}`;
|
||||
private readonly desktopPublicSession = `openclaw-desktop-${randomUUID()}`;
|
||||
private windowStartPromise: Promise<void> | undefined;
|
||||
private desktopStartPromise: Promise<void> | undefined;
|
||||
private windowStarted = false;
|
||||
private desktopStarted = false;
|
||||
private disposed = false;
|
||||
|
||||
constructor(private readonly sdk: CuaDriverSdk) {
|
||||
@@ -108,77 +114,66 @@ class DirectCuaDriverSession implements CuaDriverSession {
|
||||
maxIdleTtlSeconds: 300n,
|
||||
};
|
||||
// Never use CuaDriver.create(): configured creation fixes the authorization
|
||||
// ceiling before a single trusted OpenClaw session is admitted.
|
||||
// ceiling before the paired window- and desktop-scope sessions are admitted.
|
||||
this.runtime = sdk.CuaDriver.createConfigured({
|
||||
claudeCodeCompatibility: false,
|
||||
authorization,
|
||||
});
|
||||
this.session = sdk.createTrustedSession(this.runtime, {
|
||||
publicSession: this.publicSession,
|
||||
const sessionOptions = {
|
||||
mode: unrestricted,
|
||||
ttlSeconds: authorization.maxSessionTtlSeconds,
|
||||
idleTtlSeconds: authorization.maxIdleTtlSeconds,
|
||||
};
|
||||
this.windowSession = sdk.createTrustedSession(this.runtime, {
|
||||
...sessionOptions,
|
||||
publicSession: this.windowPublicSession,
|
||||
});
|
||||
this.desktopSession = sdk.createTrustedSession(this.runtime, {
|
||||
...sessionOptions,
|
||||
publicSession: this.desktopPublicSession,
|
||||
});
|
||||
}
|
||||
|
||||
private async ensureStarted(
|
||||
captureScope: DriverCaptureScope,
|
||||
private async ensureSessionStarted(
|
||||
kind: "window" | "desktop",
|
||||
signal?: AbortSignal,
|
||||
): Promise<void> {
|
||||
if (this.disposed) {
|
||||
throw new Error("COMPUTER_DRIVER_UNAVAILABLE: cua-computer is stopping");
|
||||
}
|
||||
if (!this.startPromise) {
|
||||
this.captureScope = captureScope;
|
||||
const start = this.session
|
||||
.startSession({ session: this.publicSession, captureScope }, asyncOptions(signal))
|
||||
const isWindow = kind === "window";
|
||||
const session = isWindow ? this.windowSession : this.desktopSession;
|
||||
const publicSession = isWindow ? this.windowPublicSession : this.desktopPublicSession;
|
||||
const captureScope = isWindow ? this.sdk.CaptureScope.Window : this.sdk.CaptureScope.Desktop;
|
||||
const startedKey = isWindow ? "windowStarted" : "desktopStarted";
|
||||
const startPromiseKey = isWindow ? "windowStartPromise" : "desktopStartPromise";
|
||||
const current = this[startPromiseKey];
|
||||
if (!current) {
|
||||
const start = session
|
||||
.startSession({ session: publicSession, captureScope }, asyncOptions(signal))
|
||||
.then(() => {
|
||||
this.started = true;
|
||||
this[startedKey] = true;
|
||||
});
|
||||
this.startPromise = start;
|
||||
this[startPromiseKey] = start;
|
||||
try {
|
||||
await start;
|
||||
} catch (error) {
|
||||
if (this.startPromise === start) {
|
||||
this.startPromise = undefined;
|
||||
if (this[startPromiseKey] === start) {
|
||||
this[startPromiseKey] = undefined;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
return;
|
||||
}
|
||||
await this.startPromise;
|
||||
if (
|
||||
captureScope === this.sdk.CaptureScope.Desktop &&
|
||||
this.captureScope !== this.sdk.CaptureScope.Desktop
|
||||
) {
|
||||
await this.ensureDesktopScope(signal);
|
||||
}
|
||||
}
|
||||
|
||||
private async ensureDesktopScope(signal?: AbortSignal): Promise<void> {
|
||||
if (!this.desktopEscalationPromise) {
|
||||
this.desktopEscalationPromise = this.session
|
||||
.escalateSession(
|
||||
{
|
||||
session: this.publicSession,
|
||||
reason: this.sdk.EscalationReason.Other,
|
||||
detail: "explicit desktop-scope OpenClaw action",
|
||||
},
|
||||
asyncOptions(signal),
|
||||
)
|
||||
.then(() => {
|
||||
this.captureScope = this.sdk.CaptureScope.Desktop;
|
||||
});
|
||||
}
|
||||
await this.desktopEscalationPromise;
|
||||
await current;
|
||||
}
|
||||
|
||||
private async invoke<T>(
|
||||
captureScope: DriverCaptureScope,
|
||||
kind: "window" | "desktop",
|
||||
signal: AbortSignal | undefined,
|
||||
operation: () => Promise<T>,
|
||||
): Promise<T> {
|
||||
await this.ensureStarted(captureScope, signal);
|
||||
await this.ensureSessionStarted(kind, signal);
|
||||
return await operation();
|
||||
}
|
||||
|
||||
@@ -187,52 +182,65 @@ class DirectCuaDriverSession implements CuaDriverSession {
|
||||
}
|
||||
resetAvailabilityCache(): void {}
|
||||
async callTool(name: string, args: Record<string, unknown>, signal?: AbortSignal) {
|
||||
return await this.invoke(this.sdk.CaptureScope.Window, signal, () =>
|
||||
this.session.callTool(
|
||||
return await this.invoke("window", signal, () =>
|
||||
this.windowSession.callTool(
|
||||
name,
|
||||
JSON.stringify({ ...args, session: this.publicSession }),
|
||||
JSON.stringify({ ...args, session: this.windowPublicSession }),
|
||||
asyncOptions(signal),
|
||||
),
|
||||
);
|
||||
}
|
||||
async escalateScope(reason: EscalationReason, signal?: AbortSignal) {
|
||||
await this.ensureStarted(this.sdk.CaptureScope.Window, signal);
|
||||
const state = await this.session.escalateSession(
|
||||
{ session: this.publicSession, reason },
|
||||
async callDesktopTool(name: string, args: Record<string, unknown>, signal?: AbortSignal) {
|
||||
return await this.invoke("desktop", signal, () =>
|
||||
this.desktopSession.callTool(
|
||||
name,
|
||||
JSON.stringify({ ...args, session: this.desktopPublicSession }),
|
||||
asyncOptions(signal),
|
||||
),
|
||||
);
|
||||
}
|
||||
async escalateScope(_reason: EscalationReason, signal?: AbortSignal) {
|
||||
await this.ensureSessionStarted("desktop", signal);
|
||||
return await this.desktopSession.getSessionState(
|
||||
{ session: this.desktopPublicSession },
|
||||
asyncOptions(signal),
|
||||
);
|
||||
this.captureScope = this.sdk.CaptureScope.Desktop;
|
||||
return state;
|
||||
}
|
||||
async getDesktopState(signal?: AbortSignal) {
|
||||
return await this.invoke(this.sdk.CaptureScope.Desktop, signal, () =>
|
||||
this.session.getDesktopState({}, asyncOptions(signal)),
|
||||
return await this.invoke("desktop", signal, () =>
|
||||
this.desktopSession.getDesktopState({}, asyncOptions(signal)),
|
||||
);
|
||||
}
|
||||
async getScreenSize(signal?: AbortSignal) {
|
||||
return await this.invoke(this.sdk.CaptureScope.Desktop, signal, () =>
|
||||
this.session.getScreenSize({}, asyncOptions(signal)),
|
||||
return await this.invoke("desktop", signal, () =>
|
||||
this.desktopSession.getScreenSize({}, asyncOptions(signal)),
|
||||
);
|
||||
}
|
||||
async click(
|
||||
input: { x: number; y: number; button: ClickButton; count: number },
|
||||
signal?: AbortSignal,
|
||||
) {
|
||||
return await this.invoke(this.sdk.CaptureScope.Desktop, signal, () =>
|
||||
this.session.click({ ...input, scope: this.sdk.DesktopScope.Desktop }, asyncOptions(signal)),
|
||||
return await this.invoke("desktop", signal, () =>
|
||||
this.desktopSession.click(
|
||||
{ ...input, scope: this.sdk.DesktopScope.Desktop },
|
||||
asyncOptions(signal),
|
||||
),
|
||||
);
|
||||
}
|
||||
async drag(
|
||||
input: { fromX: number; fromY: number; toX: number; toY: number; durationMs?: bigint },
|
||||
signal?: AbortSignal,
|
||||
) {
|
||||
return await this.invoke(this.sdk.CaptureScope.Desktop, signal, () =>
|
||||
this.session.drag({ ...input, scope: this.sdk.DesktopScope.Desktop }, asyncOptions(signal)),
|
||||
return await this.invoke("desktop", signal, () =>
|
||||
this.desktopSession.drag(
|
||||
{ ...input, scope: this.sdk.DesktopScope.Desktop },
|
||||
asyncOptions(signal),
|
||||
),
|
||||
);
|
||||
}
|
||||
async moveCursor(input: { x: number; y: number }, signal?: AbortSignal) {
|
||||
return await this.invoke(this.sdk.CaptureScope.Desktop, signal, () =>
|
||||
this.session.moveCursor(
|
||||
return await this.invoke("desktop", signal, () =>
|
||||
this.desktopSession.moveCursor(
|
||||
{ ...input, scope: this.sdk.DesktopScope.Desktop },
|
||||
asyncOptions(signal),
|
||||
),
|
||||
@@ -242,8 +250,8 @@ class DirectCuaDriverSession implements CuaDriverSession {
|
||||
input: { x: number; y: number; direction: ScrollDirection; amount: bigint },
|
||||
signal?: AbortSignal,
|
||||
) {
|
||||
return await this.invoke(this.sdk.CaptureScope.Desktop, signal, () =>
|
||||
this.session.scroll(
|
||||
return await this.invoke("desktop", signal, () =>
|
||||
this.desktopSession.scroll(
|
||||
{
|
||||
...input,
|
||||
scope: this.sdk.DesktopScope.Desktop,
|
||||
@@ -254,13 +262,16 @@ class DirectCuaDriverSession implements CuaDriverSession {
|
||||
);
|
||||
}
|
||||
async typeText(text: string, signal?: AbortSignal) {
|
||||
return await this.invoke(this.sdk.CaptureScope.Desktop, signal, () =>
|
||||
this.session.typeText({ text, scope: this.sdk.DesktopScope.Desktop }, asyncOptions(signal)),
|
||||
return await this.invoke("desktop", signal, () =>
|
||||
this.desktopSession.typeText(
|
||||
{ text, scope: this.sdk.DesktopScope.Desktop },
|
||||
asyncOptions(signal),
|
||||
),
|
||||
);
|
||||
}
|
||||
async pressKey(input: { key: string; modifiers: string[] }, signal?: AbortSignal) {
|
||||
return await this.invoke(this.sdk.CaptureScope.Desktop, signal, () =>
|
||||
this.session.pressKey(
|
||||
return await this.invoke("desktop", signal, () =>
|
||||
this.desktopSession.pressKey(
|
||||
{ ...input, scope: this.sdk.DesktopScope.Desktop },
|
||||
asyncOptions(signal),
|
||||
),
|
||||
@@ -273,24 +284,37 @@ class DirectCuaDriverSession implements CuaDriverSession {
|
||||
}
|
||||
this.disposed = true;
|
||||
let failure: unknown;
|
||||
try {
|
||||
await this.startPromise;
|
||||
} catch (error) {
|
||||
failure = error;
|
||||
}
|
||||
if (this.started) {
|
||||
for (const entry of [
|
||||
{
|
||||
session: this.windowSession,
|
||||
publicSession: this.windowPublicSession,
|
||||
start: this.windowStartPromise,
|
||||
started: this.windowStarted,
|
||||
},
|
||||
{
|
||||
session: this.desktopSession,
|
||||
publicSession: this.desktopPublicSession,
|
||||
start: this.desktopStartPromise,
|
||||
started: this.desktopStarted,
|
||||
},
|
||||
]) {
|
||||
try {
|
||||
// End the native desktop session before revoking its trusted handle.
|
||||
// Closing only the handle can leave the started session behind on stop.
|
||||
await this.session.endSession({ session: this.publicSession });
|
||||
await entry.start;
|
||||
} catch (error) {
|
||||
failure ??= error;
|
||||
}
|
||||
if (entry.started) {
|
||||
try {
|
||||
await entry.session.endSession({ session: entry.publicSession });
|
||||
} catch (error) {
|
||||
failure ??= error;
|
||||
}
|
||||
}
|
||||
try {
|
||||
entry.session.close();
|
||||
} catch (error) {
|
||||
failure ??= error;
|
||||
}
|
||||
}
|
||||
try {
|
||||
this.session.close();
|
||||
} catch (error) {
|
||||
failure = error;
|
||||
}
|
||||
try {
|
||||
await this.runtime.shutdown();
|
||||
@@ -424,6 +448,9 @@ class LazyCuaDriverSession implements CuaDriverSession {
|
||||
async callTool(name: string, args: Record<string, unknown>, signal?: AbortSignal) {
|
||||
return await (await this.requireRuntime()).callTool(name, args, signal);
|
||||
}
|
||||
async callDesktopTool(name: string, args: Record<string, unknown>, signal?: AbortSignal) {
|
||||
return await (await this.requireRuntime()).callDesktopTool(name, args, signal);
|
||||
}
|
||||
async escalateScope(reason: EscalationReason, signal?: AbortSignal) {
|
||||
return await (await this.requireRuntime()).escalateScope(reason, signal);
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ import net from "node:net";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { ClickButton } from "./driver-client.js";
|
||||
import { ClickButton, EscalationReason } from "./driver-client.js";
|
||||
import { createCuaMcpDriver } from "./mcp-driver-client.js";
|
||||
|
||||
type RpcRequest = {
|
||||
@@ -177,6 +177,24 @@ describe.runIf(process.platform !== "win32")("CUA MCP proxy transport", () => {
|
||||
}),
|
||||
);
|
||||
break;
|
||||
case "browser_navigate":
|
||||
fake.respond(
|
||||
request,
|
||||
toolResult({
|
||||
status: "ok",
|
||||
target_id: "target-1",
|
||||
tab_id: "tab-1",
|
||||
url: "https://example.com/",
|
||||
refs_invalidated: true,
|
||||
}),
|
||||
);
|
||||
break;
|
||||
case "list_windows":
|
||||
fake.respond(request, toolResult({ windows: [] }));
|
||||
break;
|
||||
case "get_session_state":
|
||||
fake.respond(request, sessionState("desktop"));
|
||||
break;
|
||||
case "end_session":
|
||||
fake.respond(request, toolResult({ session: "openclaw-test", active: false }));
|
||||
break;
|
||||
@@ -209,6 +227,52 @@ describe.runIf(process.platform !== "win32")("CUA MCP proxy transport", () => {
|
||||
delivery: { mode: 0, deliveredCount: 1 },
|
||||
evidence: [{ kind: 0 }],
|
||||
});
|
||||
await driver.callTool("browser_navigate", {
|
||||
target_id: "target-1",
|
||||
tab_id: "tab-1",
|
||||
url: "https://example.com/",
|
||||
});
|
||||
await expect(driver.callTool("list_windows", {})).resolves.toMatchObject({
|
||||
isError: false,
|
||||
});
|
||||
await driver.escalateScope(EscalationReason.Other);
|
||||
await expect(driver.callTool("list_windows", {})).resolves.toMatchObject({
|
||||
isError: false,
|
||||
});
|
||||
|
||||
const startCalls = endpoint.requests.filter(
|
||||
(request) => request.method === "tools/call" && request.params?.name === "start_session",
|
||||
);
|
||||
expect(startCalls).toHaveLength(2);
|
||||
const captureScopes = startCalls.flatMap((request) => {
|
||||
const scope = request.params?.arguments?.capture_scope;
|
||||
return typeof scope === "string" ? [scope] : [];
|
||||
});
|
||||
expect(captureScopes.toSorted((left, right) => left.localeCompare(right))).toEqual([
|
||||
"desktop",
|
||||
"window",
|
||||
]);
|
||||
expect(new Set(startCalls.map((request) => request.params?.arguments?.session)).size).toBe(2);
|
||||
const desktopSession = startCalls.find(
|
||||
(request) => request.params?.arguments?.capture_scope === "desktop",
|
||||
)?.params?.arguments?.session;
|
||||
const windowSession = startCalls.find(
|
||||
(request) => request.params?.arguments?.capture_scope === "window",
|
||||
)?.params?.arguments?.session;
|
||||
expect(
|
||||
endpoint.requests.find(
|
||||
(request) =>
|
||||
request.method === "tools/call" && request.params?.name === "get_session_state",
|
||||
)?.params?.arguments?.session,
|
||||
).toBe(desktopSession);
|
||||
expect(
|
||||
endpoint.requests
|
||||
.filter(
|
||||
(request) => request.method === "tools/call" && request.params?.name === "list_windows",
|
||||
)
|
||||
.map((request) => request.params?.arguments?.session),
|
||||
).toEqual([windowSession, windowSession]);
|
||||
|
||||
await driver.dispose();
|
||||
await vi.waitFor(() => {
|
||||
closed = endpoint.requests.some(
|
||||
@@ -245,6 +309,47 @@ describe.runIf(process.platform !== "win32")("CUA MCP proxy transport", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("ends a started window session when desktop startup fails", async () => {
|
||||
let desktopStart: RpcRequest | undefined;
|
||||
const endedSessions: unknown[] = [];
|
||||
const endpoint = await createFakeEndpoint((request, fake) => {
|
||||
if (request.method === "initialize") {
|
||||
fake.respond(request, {
|
||||
protocolVersion: "2025-06-18",
|
||||
capabilities: { tools: {} },
|
||||
serverInfo: { name: "fake-cua-driver", version: "0.19.3" },
|
||||
});
|
||||
} else if (request.method === "tools/call" && request.params?.name === "start_session") {
|
||||
if (request.params.arguments?.capture_scope === "desktop") {
|
||||
desktopStart = request;
|
||||
} else {
|
||||
fake.respond(request, sessionState("window"));
|
||||
}
|
||||
} else if (request.method === "tools/call" && request.params?.name === "list_windows") {
|
||||
fake.respond(request, toolResult({ windows: [] }));
|
||||
} else if (request.method === "tools/call" && request.params?.name === "end_session") {
|
||||
endedSessions.push(request.params.arguments?.session);
|
||||
fake.respond(request, toolResult({ session: request.params.arguments?.session }));
|
||||
}
|
||||
});
|
||||
try {
|
||||
const driver = createCuaMcpDriver(endpoint);
|
||||
await vi.waitFor(() => expect(driver.isAvailable()).toBe(true));
|
||||
await driver.callTool("list_windows", {});
|
||||
const desktopCall = driver.getDesktopState().catch((error: unknown) => error);
|
||||
await vi.waitFor(() => expect(desktopStart).toBeDefined());
|
||||
const disposeCall = driver.dispose().catch((error: unknown) => error);
|
||||
endpoint.respond(desktopStart!, { ...toolResult({}), isError: true });
|
||||
|
||||
await expect(desktopCall).resolves.toBeInstanceOf(Error);
|
||||
await expect(disposeCall).resolves.toBeInstanceOf(Error);
|
||||
expect(endedSessions).toHaveLength(1);
|
||||
expect(endedSessions[0]).toEqual(expect.stringMatching(/^openclaw-window-/));
|
||||
} finally {
|
||||
await endpoint.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("bounds pending calls and tears down the proxy on cancellation", async () => {
|
||||
const held: RpcRequest[] = [];
|
||||
const endpoint = await createFakeEndpoint((request, fake) => {
|
||||
|
||||
@@ -484,10 +484,12 @@ function sessionState(value: CuaToolResult): import("@trycua/cua-driver").Sessio
|
||||
|
||||
class McpCuaDriverSession implements CuaDriverSession {
|
||||
readonly generation = randomUUID();
|
||||
private readonly publicSession = `openclaw-${randomUUID()}`;
|
||||
private startPromise: Promise<void> | undefined;
|
||||
private captureScope: "window" | "desktop" | undefined;
|
||||
private started = false;
|
||||
private readonly windowPublicSession = `openclaw-window-${randomUUID()}`;
|
||||
private readonly desktopPublicSession = `openclaw-desktop-${randomUUID()}`;
|
||||
private windowStartPromise: Promise<void> | undefined;
|
||||
private desktopStartPromise: Promise<void> | undefined;
|
||||
private windowStarted = false;
|
||||
private desktopStarted = false;
|
||||
private disposed = false;
|
||||
|
||||
constructor(private readonly client: CuaMcpProxyClient) {}
|
||||
@@ -500,26 +502,20 @@ class McpCuaDriverSession implements CuaDriverSession {
|
||||
|
||||
async callTool(name: string, args: Record<string, unknown>, signal?: AbortSignal) {
|
||||
await this.ensureStarted("window", signal);
|
||||
return await this.client.callTool(name, { ...args, session: this.publicSession }, signal);
|
||||
return await this.client.callTool(name, { ...args, session: this.windowPublicSession }, signal);
|
||||
}
|
||||
|
||||
async escalateScope(reason: EscalationReason, signal?: AbortSignal) {
|
||||
await this.ensureStarted("window", signal);
|
||||
async callDesktopTool(name: string, args: Record<string, unknown>, signal?: AbortSignal) {
|
||||
return await this.desktopTool(name, args, signal);
|
||||
}
|
||||
|
||||
async escalateScope(_reason: EscalationReason, signal?: AbortSignal) {
|
||||
await this.ensureStarted("desktop", signal);
|
||||
const result = await this.client.callTool(
|
||||
"escalate_session",
|
||||
{
|
||||
session: this.publicSession,
|
||||
reason: [
|
||||
"ax_tree_pixel_mismatch",
|
||||
"background_delivery_failed",
|
||||
"foreground_ineffective",
|
||||
"no_window_target",
|
||||
"other",
|
||||
][reason],
|
||||
},
|
||||
"get_session_state",
|
||||
{ session: this.desktopPublicSession },
|
||||
signal,
|
||||
);
|
||||
this.captureScope = "desktop";
|
||||
return sessionState(result);
|
||||
}
|
||||
|
||||
@@ -610,13 +606,30 @@ class McpCuaDriverSession implements CuaDriverSession {
|
||||
}
|
||||
this.disposed = true;
|
||||
let failure: unknown;
|
||||
try {
|
||||
await this.startPromise;
|
||||
if (this.started && this.client.isAvailable()) {
|
||||
await this.client.callTool("end_session", { session: this.publicSession });
|
||||
const startResults = await Promise.allSettled([
|
||||
this.windowStartPromise,
|
||||
this.desktopStartPromise,
|
||||
]);
|
||||
for (const result of startResults) {
|
||||
if (result.status === "rejected") {
|
||||
failure ??= result.reason;
|
||||
}
|
||||
}
|
||||
if (this.client.isAvailable()) {
|
||||
if (this.windowStarted) {
|
||||
try {
|
||||
await this.client.callTool("end_session", { session: this.windowPublicSession });
|
||||
} catch (error) {
|
||||
failure ??= error;
|
||||
}
|
||||
}
|
||||
if (this.desktopStarted) {
|
||||
try {
|
||||
await this.client.callTool("end_session", { session: this.desktopPublicSession });
|
||||
} catch (error) {
|
||||
failure ??= error;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
failure = error;
|
||||
}
|
||||
try {
|
||||
await this.client.stop();
|
||||
@@ -636,38 +649,43 @@ class McpCuaDriverSession implements CuaDriverSession {
|
||||
signal?: AbortSignal,
|
||||
): Promise<CuaToolResult> {
|
||||
await this.ensureStarted("desktop", signal);
|
||||
return await this.client.callTool(name, { ...args, session: this.publicSession }, signal);
|
||||
return await this.client.callTool(
|
||||
name,
|
||||
{ ...args, session: this.desktopPublicSession },
|
||||
signal,
|
||||
);
|
||||
}
|
||||
|
||||
private async ensureStarted(scope: "window" | "desktop", signal?: AbortSignal): Promise<void> {
|
||||
if (this.disposed) {
|
||||
throw driverUnavailable("cua-computer is stopping");
|
||||
}
|
||||
if (!this.startPromise) {
|
||||
this.captureScope = scope;
|
||||
const isWindow = scope === "window";
|
||||
const startedKey = isWindow ? "windowStarted" : "desktopStarted";
|
||||
const startPromiseKey = isWindow ? "windowStartPromise" : "desktopStartPromise";
|
||||
const current = this[startPromiseKey];
|
||||
if (!current) {
|
||||
const publicSession = isWindow ? this.windowPublicSession : this.desktopPublicSession;
|
||||
const start = this.client
|
||||
.callTool("start_session", { session: this.publicSession, capture_scope: scope }, signal)
|
||||
.callTool("start_session", { session: publicSession, capture_scope: scope }, signal)
|
||||
.then((result) => {
|
||||
if (result.isError) {
|
||||
throw driverProtocolError(result.text || "CUA MCP start_session failed");
|
||||
}
|
||||
this.started = true;
|
||||
this[startedKey] = true;
|
||||
});
|
||||
this.startPromise = start;
|
||||
this[startPromiseKey] = start;
|
||||
try {
|
||||
await start;
|
||||
} catch (error) {
|
||||
if (this.startPromise === start) {
|
||||
this.startPromise = undefined;
|
||||
if (this[startPromiseKey] === start) {
|
||||
this[startPromiseKey] = undefined;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
return;
|
||||
}
|
||||
await this.startPromise;
|
||||
if (scope === "desktop" && this.captureScope !== "desktop") {
|
||||
await this.escalateScope(EscalationReason.Other, signal);
|
||||
}
|
||||
await current;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -272,7 +272,7 @@ export async function handleV2Act(
|
||||
});
|
||||
}
|
||||
case "get_cursor_position": {
|
||||
const result = await callWindowTool(driver, state, "get_cursor_position", {}, signal);
|
||||
const result = await driver.callDesktopTool("get_cursor_position", {}, signal);
|
||||
return JSON.stringify({
|
||||
ok: true,
|
||||
details: projectedToolDetails(result, "get_cursor_position"),
|
||||
|
||||
@@ -0,0 +1,334 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { parseArgs, promisify } from "node:util";
|
||||
import { createComputerTool } from "../../src/agents/tools/computer-tool.js";
|
||||
import { listNodes } from "../../src/agents/tools/nodes-utils.js";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const { values } = parseArgs({
|
||||
options: {
|
||||
"window-title": { type: "string" },
|
||||
provider: { type: "string" },
|
||||
text: { type: "string" },
|
||||
artifacts: { type: "string" },
|
||||
"element-label": { type: "string" },
|
||||
help: { type: "boolean", short: "h" },
|
||||
},
|
||||
strict: true,
|
||||
});
|
||||
|
||||
if (values.help) {
|
||||
console.log(
|
||||
"Usage: computer-use-macos-live-proof.ts --provider <peekaboo|cua> --window-title <title> --text <text> --artifacts <dir> [--element-label <label>]",
|
||||
);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const windowTitle = values["window-title"]?.trim();
|
||||
const provider = values.provider?.trim().toLowerCase();
|
||||
const text = values.text;
|
||||
const artifacts = values.artifacts ? path.resolve(values.artifacts) : undefined;
|
||||
const elementLabel = values["element-label"]?.trim().toLowerCase();
|
||||
if (
|
||||
(provider !== "peekaboo" && provider !== "cua") ||
|
||||
!windowTitle ||
|
||||
text === undefined ||
|
||||
!artifacts
|
||||
) {
|
||||
throw new Error(
|
||||
"--provider (peekaboo|cua), --window-title, --text, and --artifacts are required",
|
||||
);
|
||||
}
|
||||
const artifactDirectory = artifacts;
|
||||
|
||||
type ToolResult = Awaited<ReturnType<ReturnType<typeof createComputerTool>["execute"]>>;
|
||||
type JsonRecord = Record<string, unknown>;
|
||||
type ActionOutcome = { kind: "result"; result: ToolResult } | { kind: "error"; error: JsonRecord };
|
||||
|
||||
const expectedProviderId = provider === "cua" ? "cua-computer" : "peekaboo";
|
||||
const computerNodes = (await listNodes({ timeoutMs: 30_000 })).filter(
|
||||
(node) =>
|
||||
node.connected === true &&
|
||||
node.commands?.includes("computer.act") === true &&
|
||||
node.commands.includes("screen.snapshot") &&
|
||||
node.computerUse !== undefined,
|
||||
);
|
||||
if (computerNodes.length !== 1) {
|
||||
throw new Error(`expected exactly one connected computer node, found ${computerNodes.length}`);
|
||||
}
|
||||
const selectedNode = computerNodes[0]!;
|
||||
const advertisedProvider = selectedNode.computerUse!.provider;
|
||||
if (advertisedProvider.id !== expectedProviderId) {
|
||||
throw new Error(
|
||||
`expected provider ${expectedProviderId}, but node advertised ${advertisedProvider.id}`,
|
||||
);
|
||||
}
|
||||
const tool = createComputerTool({
|
||||
modelHasVision: true,
|
||||
capabilityDescriptor: selectedNode.computerUse,
|
||||
});
|
||||
let callSequence = 0;
|
||||
|
||||
async function call(action: string, fields: JsonRecord = {}): Promise<ToolResult> {
|
||||
callSequence += 1;
|
||||
return await tool.execute(`live-proof-${callSequence}`, {
|
||||
action,
|
||||
node: selectedNode.nodeId,
|
||||
timeoutMs: 30_000,
|
||||
...fields,
|
||||
});
|
||||
}
|
||||
|
||||
async function attempt(action: string, fields: JsonRecord = {}): Promise<ActionOutcome> {
|
||||
try {
|
||||
return { kind: "result", result: await call(action, fields) };
|
||||
} catch (error) {
|
||||
const candidate = error as {
|
||||
name?: unknown;
|
||||
message?: unknown;
|
||||
code?: unknown;
|
||||
gatewayCode?: unknown;
|
||||
details?: unknown;
|
||||
};
|
||||
return {
|
||||
kind: "error",
|
||||
error: {
|
||||
name: candidate?.name,
|
||||
message: candidate?.message,
|
||||
code: candidate?.code,
|
||||
gatewayCode: candidate?.gatewayCode,
|
||||
details: candidate?.details,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function resultText(result: ToolResult): string {
|
||||
return result.content
|
||||
.filter((block) => block.type === "text")
|
||||
.map((block) => block.text)
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
function wireResult(result: ToolResult): JsonRecord {
|
||||
const details = result.details as { result?: unknown } | undefined;
|
||||
if (details?.result && typeof details.result === "object" && !Array.isArray(details.result)) {
|
||||
return details.result as JsonRecord;
|
||||
}
|
||||
for (const line of resultText(result).split("\n")) {
|
||||
try {
|
||||
const parsed = JSON.parse(line) as unknown;
|
||||
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
|
||||
return parsed as JsonRecord;
|
||||
}
|
||||
} catch {
|
||||
// Mutating actions prefix their follow-up screenshot with one JSON result line.
|
||||
}
|
||||
}
|
||||
throw new Error(`missing structured result: ${resultText(result)}`);
|
||||
}
|
||||
|
||||
function record(value: unknown): JsonRecord | undefined {
|
||||
return value !== null && typeof value === "object" && !Array.isArray(value)
|
||||
? (value as JsonRecord)
|
||||
: undefined;
|
||||
}
|
||||
|
||||
function records(value: unknown): JsonRecord[] {
|
||||
return Array.isArray(value) ? value.map(record).filter((entry) => entry !== undefined) : [];
|
||||
}
|
||||
|
||||
function stringValue(value: unknown): string {
|
||||
return typeof value === "string" ? value : "";
|
||||
}
|
||||
|
||||
function summarizeResult(result: ToolResult): JsonRecord {
|
||||
const raw = wireResult(result);
|
||||
const observed = record(raw.observation);
|
||||
const details = record(raw.details);
|
||||
return {
|
||||
action: raw.action,
|
||||
ok: raw.ok,
|
||||
effect: raw.effect,
|
||||
error: raw.error,
|
||||
details,
|
||||
observation: observed
|
||||
? {
|
||||
kind: observed.kind,
|
||||
format: observed.format,
|
||||
width: observed.width,
|
||||
height: observed.height,
|
||||
observationId: observed.observationId,
|
||||
elements: records(observed.elements).map((element) => ({
|
||||
elementRef: element.elementRef,
|
||||
role: element.role,
|
||||
label: element.label,
|
||||
value: element.value,
|
||||
bounds: element.bounds,
|
||||
})),
|
||||
}
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeOutcome(outcome: ActionOutcome): JsonRecord {
|
||||
return outcome.kind === "result" ? summarizeResult(outcome.result) : { error: outcome.error };
|
||||
}
|
||||
|
||||
async function saveImage(name: string, result: ToolResult): Promise<string> {
|
||||
const image = result.content.find((block) => block.type === "image");
|
||||
if (!image || image.type !== "image") {
|
||||
throw new Error(`missing model-visible image in ${name}`);
|
||||
}
|
||||
const extension = image.mimeType === "image/jpeg" ? "jpeg" : "png";
|
||||
const output = path.join(artifactDirectory, `${name}.${extension}`);
|
||||
await writeFile(output, Buffer.from(image.data, "base64"));
|
||||
return output;
|
||||
}
|
||||
|
||||
async function frontmostApp(): Promise<string> {
|
||||
const script =
|
||||
'tell application "System Events" to get name of first application process whose frontmost is true';
|
||||
const { stdout } = await execFileAsync("/usr/bin/osascript", ["-e", script]);
|
||||
return stdout.trim();
|
||||
}
|
||||
|
||||
function cursor(result: ToolResult): { x: unknown; y: unknown } {
|
||||
const details = record(wireResult(result).details);
|
||||
return { x: details?.x, y: details?.y };
|
||||
}
|
||||
|
||||
function observation(result: ToolResult): JsonRecord {
|
||||
const value = record(wireResult(result).observation);
|
||||
if (!value) {
|
||||
throw new Error(`missing observation: ${resultText(result)}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function selectElement(result: ToolResult): JsonRecord {
|
||||
const elements = records(observation(result).elements);
|
||||
const editable = elements.filter((element) =>
|
||||
["AXTextArea", "AXTextField", "text_area", "text_field"].includes(stringValue(element.role)),
|
||||
);
|
||||
const selected = elementLabel
|
||||
? editable.find((element) => stringValue(element.label).toLowerCase().includes(elementLabel))
|
||||
: editable[0];
|
||||
if (!selected) {
|
||||
throw new Error(`no editable element matched ${elementLabel ?? "the target window"}`);
|
||||
}
|
||||
return selected;
|
||||
}
|
||||
|
||||
function structuredOutcome(outcome: ActionOutcome): boolean {
|
||||
if (outcome.kind === "error") {
|
||||
return typeof outcome.error.code === "string" || typeof outcome.error.gatewayCode === "string";
|
||||
}
|
||||
const raw = wireResult(outcome.result);
|
||||
if (raw.effect === "confirmed") {
|
||||
return true;
|
||||
}
|
||||
const error = record(raw.error);
|
||||
return raw.ok === false && typeof error?.code === "string" && error.code.length > 0;
|
||||
}
|
||||
|
||||
await mkdir(artifactDirectory, { recursive: true });
|
||||
const screenshot = await call("screenshot");
|
||||
const listed = await call("list_windows");
|
||||
const windows = records(record(wireResult(listed).details)?.windows);
|
||||
const target = windows.find((window) => stringValue(window.title).includes(windowTitle));
|
||||
if (!target || typeof target.windowRef !== "string") {
|
||||
throw new Error(`window containing ${JSON.stringify(windowTitle)} was not found`);
|
||||
}
|
||||
|
||||
const before = await call("get_window_state", { windowRef: target.windowRef });
|
||||
const beforeImage = await saveImage("window-before", before);
|
||||
const beforeObservation = observation(before);
|
||||
const beforeElement = selectElement(before);
|
||||
const observationId = beforeObservation.observationId;
|
||||
if (typeof observationId !== "string" || typeof beforeElement.elementRef !== "string") {
|
||||
throw new Error("target observation did not provide stable element references");
|
||||
}
|
||||
|
||||
const targetFields = {
|
||||
windowRef: target.windowRef,
|
||||
elementRef: beforeElement.elementRef,
|
||||
observationId,
|
||||
deliveryMode: "background",
|
||||
};
|
||||
const frontmostBefore = await frontmostApp();
|
||||
if (frontmostBefore === target.appName) {
|
||||
throw new Error(
|
||||
`target app ${stringValue(target.appName) || "<unknown>"} is frontmost; foreground another app and retry`,
|
||||
);
|
||||
}
|
||||
const cursorBeforeResult = await call("get_cursor_position");
|
||||
const click = await attempt("left_click", targetFields);
|
||||
const typed = await attempt("type", { ...targetFields, text });
|
||||
|
||||
let confirmation: ActionOutcome | undefined;
|
||||
if (typed.kind === "error" || wireResult(typed.result).effect !== "confirmed") {
|
||||
const refreshed = await call("get_window_state", { windowRef: target.windowRef });
|
||||
const refreshedObservation = observation(refreshed);
|
||||
const refreshedElement = selectElement(refreshed);
|
||||
confirmation = await attempt("set_value", {
|
||||
windowRef: target.windowRef,
|
||||
elementRef: refreshedElement.elementRef,
|
||||
observationId: refreshedObservation.observationId,
|
||||
deliveryMode: "background",
|
||||
value: text,
|
||||
});
|
||||
}
|
||||
|
||||
const cursorAfterResult = await call("get_cursor_position");
|
||||
const frontmostAfter = await frontmostApp();
|
||||
const after = await call("get_window_state", { windowRef: target.windowRef });
|
||||
const afterImage = await saveImage("window-after", after);
|
||||
const afterElement = selectElement(after);
|
||||
const cursorBefore = cursor(cursorBeforeResult);
|
||||
const cursorAfter = cursor(cursorAfterResult);
|
||||
const finalOutcome = confirmation ?? typed;
|
||||
|
||||
const evidence = {
|
||||
route: "agent computer tool -> Gateway node.invoke -> paired Mac node -> selected provider",
|
||||
provider: { expected: expectedProviderId, advertised: advertisedProvider },
|
||||
screenshot: resultText(screenshot),
|
||||
target: {
|
||||
windowRef: target.windowRef,
|
||||
appName: target.appName,
|
||||
title: target.title,
|
||||
bounds: target.bounds,
|
||||
},
|
||||
frontmost: { before: frontmostBefore, after: frontmostAfter },
|
||||
cursor: { before: cursorBefore, after: cursorAfter },
|
||||
values: { before: beforeElement.value, after: afterElement.value },
|
||||
results: {
|
||||
listWindows: {
|
||||
action: wireResult(listed).action,
|
||||
ok: wireResult(listed).ok,
|
||||
windowCount: windows.length,
|
||||
},
|
||||
before: summarizeResult(before),
|
||||
click: summarizeOutcome(click),
|
||||
type: summarizeOutcome(typed),
|
||||
confirmation: confirmation ? summarizeOutcome(confirmation) : undefined,
|
||||
after: summarizeResult(after),
|
||||
},
|
||||
artifacts: { beforeImage, afterImage },
|
||||
assertions: {
|
||||
targetWasNotFrontmost: frontmostBefore !== target.appName,
|
||||
frontmostUnchanged: frontmostBefore === frontmostAfter,
|
||||
cursorUnchanged: cursorBefore.x === cursorAfter.x && cursorBefore.y === cursorAfter.y,
|
||||
targetContentChanged: beforeElement.value !== afterElement.value,
|
||||
confirmedEffectOrStructuredRefusal: structuredOutcome(finalOutcome),
|
||||
},
|
||||
};
|
||||
|
||||
const output = path.join(artifactDirectory, "result.json");
|
||||
await writeFile(output, `${JSON.stringify(evidence, null, 2)}\n`);
|
||||
console.log(JSON.stringify(evidence, null, 2));
|
||||
|
||||
if (!Object.values(evidence.assertions).every(Boolean)) {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
Executable
+250
@@ -0,0 +1,250 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage:
|
||||
scripts/dev/computer-use-macos-live-rig.sh prepare <profile> <port> <app> <scratch> [peekaboo|cua]
|
||||
scripts/dev/computer-use-macos-live-rig.sh gateway <scratch>
|
||||
scripts/dev/computer-use-macos-live-rig.sh app <scratch> [peekaboo|cua]
|
||||
scripts/dev/computer-use-macos-live-rig.sh nodes <scratch>
|
||||
scripts/dev/computer-use-macos-live-rig.sh proof <scratch> <peekaboo|cua> <window-title> <text> [element-label]
|
||||
|
||||
The rig is maintainer-only and loopback-only. Run gateway and app in separate
|
||||
terminals, approve the dedicated CLI device after its first `nodes` request,
|
||||
then run `proof`. Never use the operator profile or port 18789.
|
||||
EOF
|
||||
}
|
||||
|
||||
fail() {
|
||||
echo "computer-use live rig: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
validate_provider() {
|
||||
case "$1" in
|
||||
peekaboo | cua) ;;
|
||||
*) fail "provider must be peekaboo or cua" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
require_unoccupied_port() {
|
||||
local port="$1"
|
||||
if /usr/sbin/lsof -nP -iTCP:"$port" -sTCP:LISTEN -t >/dev/null 2>&1; then
|
||||
fail "port $port already has a listener; choose a fresh proof port"
|
||||
fi
|
||||
}
|
||||
|
||||
load_rig() {
|
||||
local scratch="$1"
|
||||
[[ "$scratch" = /* ]] || fail "scratch path must be absolute"
|
||||
local rig_path="$scratch/rig.json"
|
||||
[[ -f "$rig_path" ]] || fail "missing $rig_path; run prepare first"
|
||||
local rig_values=()
|
||||
while IFS= read -r -d '' value; do
|
||||
rig_values+=("$value")
|
||||
done < <(node - "$rig_path" <<'NODE'
|
||||
const fs = require("node:fs");
|
||||
const path = process.argv[2];
|
||||
const keys = ["root", "profile", "port", "app", "appState", "gatewayConfig", "agentState"];
|
||||
try {
|
||||
const value = JSON.parse(fs.readFileSync(path, "utf8"));
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("expected object");
|
||||
const actualKeys = Object.keys(value).sort();
|
||||
if (actualKeys.join("\0") !== [...keys].sort().join("\0")) throw new Error("unexpected fields");
|
||||
if (!Number.isInteger(value.port)) throw new Error("port must be an integer");
|
||||
const fields = [value.root, value.profile, String(value.port), value.app, value.appState, value.gatewayConfig, value.agentState];
|
||||
if (fields.some((field) => typeof field !== "string" || field.includes("\0"))) throw new Error("invalid field");
|
||||
process.stdout.write(`${fields.join("\0")}\0`);
|
||||
} catch (error) {
|
||||
console.error(`invalid rig state: ${error instanceof Error ? error.message : String(error)}`);
|
||||
process.exit(1);
|
||||
}
|
||||
NODE
|
||||
)
|
||||
[[ ${#rig_values[@]} -eq 7 ]] || fail "invalid $rig_path"
|
||||
OPENCLAW_CU_RIG_ROOT="${rig_values[0]}"
|
||||
OPENCLAW_CU_RIG_PROFILE="${rig_values[1]}"
|
||||
OPENCLAW_CU_RIG_PORT="${rig_values[2]}"
|
||||
OPENCLAW_CU_RIG_APP="${rig_values[3]}"
|
||||
OPENCLAW_CU_RIG_APP_STATE="${rig_values[4]}"
|
||||
OPENCLAW_CU_RIG_GATEWAY_CONFIG="${rig_values[5]}"
|
||||
OPENCLAW_CU_RIG_AGENT_STATE="${rig_values[6]}"
|
||||
[[ "$OPENCLAW_CU_RIG_ROOT" == "$repo_root" ]] ||
|
||||
fail "rig belongs to a different checkout: $OPENCLAW_CU_RIG_ROOT"
|
||||
[[ "$OPENCLAW_CU_RIG_PROFILE" =~ ^[A-Za-z0-9][A-Za-z0-9_-]+$ ]] || fail "invalid rig profile"
|
||||
[[ "$OPENCLAW_CU_RIG_PORT" =~ ^[0-9]+$ ]] || fail "invalid rig port"
|
||||
((OPENCLAW_CU_RIG_PORT >= 1024 && OPENCLAW_CU_RIG_PORT <= 65535)) || fail "invalid rig port"
|
||||
((OPENCLAW_CU_RIG_PORT != 18789)) || fail "operator port is not valid rig state"
|
||||
[[ "$OPENCLAW_CU_RIG_APP" = /* ]] || fail "invalid rig app path"
|
||||
[[ "$OPENCLAW_CU_RIG_APP_STATE" == "$HOME/.openclaw-$OPENCLAW_CU_RIG_PROFILE" ]] ||
|
||||
fail "rig app state does not match its profile"
|
||||
[[ "$OPENCLAW_CU_RIG_GATEWAY_CONFIG" == "$scratch/gateway.json" ]] ||
|
||||
fail "rig gateway config is outside its scratch directory"
|
||||
[[ "$OPENCLAW_CU_RIG_AGENT_STATE" == "$scratch/agent-state" ]] ||
|
||||
fail "rig agent state is outside its scratch directory"
|
||||
}
|
||||
|
||||
prepare() {
|
||||
[[ $# -ge 4 && $# -le 5 ]] || { usage; exit 2; }
|
||||
local profile="$1"
|
||||
local port="$2"
|
||||
local app_input="$3"
|
||||
local scratch="$4"
|
||||
local provider="${5:-peekaboo}"
|
||||
|
||||
[[ "$profile" =~ ^[A-Za-z0-9][A-Za-z0-9_-]+$ ]] ||
|
||||
fail "profile must contain only letters, digits, underscores, and dashes"
|
||||
case "$profile" in
|
||||
default | main | local) fail "choose a fresh, explicitly isolated profile" ;;
|
||||
esac
|
||||
[[ "$port" =~ ^[0-9]+$ ]] || fail "port must be numeric"
|
||||
((port >= 1024 && port <= 65535)) || fail "port must be between 1024 and 65535"
|
||||
((port != 18789)) || fail "port 18789 belongs to the operator gateway"
|
||||
[[ "$scratch" = /* ]] || fail "scratch path must be absolute"
|
||||
validate_provider "$provider"
|
||||
require_unoccupied_port "$port"
|
||||
|
||||
local app_path
|
||||
app_path="$(cd "$(dirname "$app_input")" && pwd)/$(basename "$app_input")"
|
||||
local app_executable="$app_path/Contents/MacOS/OpenClaw"
|
||||
[[ -x "$app_executable" ]] || fail "signed app executable not found: $app_executable"
|
||||
codesign --verify --deep --strict "$app_path" >/dev/null 2>&1 ||
|
||||
fail "app is not a valid signed bundle: $app_path"
|
||||
|
||||
git -C "$repo_root" diff --quiet -- src packages extensions scripts/run-node.mjs scripts/run-node.mts ||
|
||||
fail "runtime sources are dirty; commit and rebuild before launching the node worker"
|
||||
git -C "$repo_root" diff --cached --quiet -- src packages extensions scripts/run-node.mjs scripts/run-node.mts ||
|
||||
fail "runtime sources are staged but uncommitted; commit and rebuild first"
|
||||
|
||||
local app_state="$HOME/.openclaw-$profile"
|
||||
local defaults_domain="ai.openclaw.mac.profile.$profile"
|
||||
[[ ! -e "$app_state" && ! -L "$app_state" ]] ||
|
||||
fail "$app_state already exists; choose a fresh proof profile"
|
||||
if defaults read "$defaults_domain" >/dev/null 2>&1; then
|
||||
fail "$defaults_domain already has saved settings; choose a fresh proof profile"
|
||||
fi
|
||||
[[ ! -e "$scratch/rig.json" ]] || fail "$scratch already contains a rig"
|
||||
mkdir -p "$scratch" "$scratch/agent-state"
|
||||
|
||||
local app_config="$app_state/openclaw.json"
|
||||
local staged_app_config="$scratch/app.json"
|
||||
local gateway_config="$scratch/gateway.json"
|
||||
|
||||
node - "$port" >"$gateway_config" <<'NODE'
|
||||
const port = Number(process.argv[2]);
|
||||
process.stdout.write(`${JSON.stringify({
|
||||
gateway: {
|
||||
mode: "local",
|
||||
port,
|
||||
auth: { mode: "none" },
|
||||
nodes: { commands: { allow: ["computer.act"] } },
|
||||
},
|
||||
}, null, 2)}\n`);
|
||||
NODE
|
||||
|
||||
node - "$port" >"$staged_app_config" <<'NODE'
|
||||
const port = Number(process.argv[2]);
|
||||
process.stdout.write(`${JSON.stringify({
|
||||
gateway: {
|
||||
mode: "remote",
|
||||
port,
|
||||
auth: { mode: "none" },
|
||||
nodes: { commands: { allow: ["computer.act"] } },
|
||||
remote: { transport: "direct", url: `ws://127.0.0.1:${port}` },
|
||||
},
|
||||
}, null, 2)}\n`);
|
||||
NODE
|
||||
|
||||
mkdir -p "$app_state"
|
||||
cp "$staged_app_config" "$app_config"
|
||||
chmod 600 "$app_config" "$gateway_config"
|
||||
|
||||
defaults write "$defaults_domain" openclaw.macNodeIdentityProfile -string node
|
||||
defaults write "$defaults_domain" openclaw.connectionMode -string remote
|
||||
defaults write "$defaults_domain" openclaw.pauseEnabled -bool false
|
||||
defaults write "$defaults_domain" openclaw.computerControlEnabled -bool true
|
||||
defaults write "$defaults_domain" openclaw.computerControlProvider -string "$provider"
|
||||
defaults write "$defaults_domain" openclaw.gatewayProjectRootPath -string "$repo_root"
|
||||
defaults write "$defaults_domain" openclaw.onboardingSeen -bool true
|
||||
defaults write "$defaults_domain" openclaw.onboardingVersion -int 8
|
||||
|
||||
node - "$repo_root" "$profile" "$port" "$app_path" "$app_state" "$gateway_config" "$scratch/agent-state" >"$scratch/rig.json" <<'NODE'
|
||||
const [root, profile, port, app, appState, gatewayConfig, agentState] = process.argv.slice(2);
|
||||
process.stdout.write(`${JSON.stringify({ root, profile, port: Number(port), app, appState, gatewayConfig, agentState }, null, 2)}\n`);
|
||||
NODE
|
||||
chmod 600 "$scratch/rig.json"
|
||||
|
||||
echo "prepared isolated profile $profile on ws://127.0.0.1:$port"
|
||||
echo "gateway: $0 gateway $scratch"
|
||||
echo "app: $0 app $scratch $provider"
|
||||
echo "nodes: $0 nodes $scratch"
|
||||
}
|
||||
|
||||
run_gateway() {
|
||||
[[ $# -eq 1 ]] || { usage; exit 2; }
|
||||
load_rig "$1"
|
||||
require_unoccupied_port "$OPENCLAW_CU_RIG_PORT"
|
||||
exec env \
|
||||
OPENCLAW_CONFIG_PATH="$OPENCLAW_CU_RIG_GATEWAY_CONFIG" \
|
||||
OPENCLAW_STATE_DIR="$OPENCLAW_CU_RIG_APP_STATE" \
|
||||
node "$repo_root/scripts/run-node.mjs" --profile "$OPENCLAW_CU_RIG_PROFILE" \
|
||||
gateway run --port "$OPENCLAW_CU_RIG_PORT" --auth none --verbose
|
||||
}
|
||||
|
||||
run_app() {
|
||||
[[ $# -ge 1 && $# -le 2 ]] || { usage; exit 2; }
|
||||
load_rig "$1"
|
||||
local provider="${2:-peekaboo}"
|
||||
validate_provider "$provider"
|
||||
defaults write "ai.openclaw.mac.profile.$OPENCLAW_CU_RIG_PROFILE" \
|
||||
openclaw.computerControlProvider -string "$provider"
|
||||
exec env OPENCLAW_PROFILE="$OPENCLAW_CU_RIG_PROFILE" \
|
||||
"$OPENCLAW_CU_RIG_APP/Contents/MacOS/OpenClaw"
|
||||
}
|
||||
|
||||
run_nodes() {
|
||||
[[ $# -eq 1 ]] || { usage; exit 2; }
|
||||
load_rig "$1"
|
||||
exec env \
|
||||
OPENCLAW_CONFIG_PATH="$OPENCLAW_CU_RIG_GATEWAY_CONFIG" \
|
||||
OPENCLAW_STATE_DIR="$OPENCLAW_CU_RIG_AGENT_STATE" \
|
||||
node "$repo_root/scripts/run-node.mjs" nodes list --json
|
||||
}
|
||||
|
||||
run_proof() {
|
||||
[[ $# -ge 4 && $# -le 5 ]] || { usage; exit 2; }
|
||||
local scratch="$1"
|
||||
load_rig "$scratch"
|
||||
local provider="$2"
|
||||
validate_provider "$provider"
|
||||
local args=(
|
||||
--provider "$provider"
|
||||
--window-title "$3"
|
||||
--text "$4"
|
||||
--artifacts "$scratch"
|
||||
)
|
||||
if [[ $# -eq 5 ]]; then
|
||||
args+=(--element-label "$5")
|
||||
fi
|
||||
exec env \
|
||||
OPENCLAW_CONFIG_PATH="$OPENCLAW_CU_RIG_GATEWAY_CONFIG" \
|
||||
OPENCLAW_STATE_DIR="$OPENCLAW_CU_RIG_AGENT_STATE" \
|
||||
node --import tsx "$repo_root/scripts/dev/computer-use-macos-live-proof.ts" "${args[@]}"
|
||||
}
|
||||
|
||||
command_name="${1:-}"
|
||||
[[ -n "$command_name" ]] || { usage; exit 2; }
|
||||
shift
|
||||
case "$command_name" in
|
||||
prepare) prepare "$@" ;;
|
||||
gateway) run_gateway "$@" ;;
|
||||
app) run_app "$@" ;;
|
||||
nodes) run_nodes "$@" ;;
|
||||
proof) run_proof "$@" ;;
|
||||
-h | --help | help) usage ;;
|
||||
*) usage; exit 2 ;;
|
||||
esac
|
||||
@@ -0,0 +1,134 @@
|
||||
/** Computer tool model-schema contract tests. */
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type {
|
||||
ComputerUseCapabilityDescriptor,
|
||||
ComputerUseV2ActionName,
|
||||
} from "../../plugins/computer-use-contract.js";
|
||||
|
||||
const { createComputerTool } = await import("./computer-tool.js");
|
||||
|
||||
type ComputerTool = ReturnType<typeof createComputerTool>;
|
||||
|
||||
function v2Descriptor(
|
||||
actions: ComputerUseV2ActionName[],
|
||||
overrides: Partial<ComputerUseCapabilityDescriptor> = {},
|
||||
): ComputerUseCapabilityDescriptor {
|
||||
return {
|
||||
contractVersion: 2 as const,
|
||||
provider: { id: "fixture", label: "Fixture", generation: "generation-1" },
|
||||
actions,
|
||||
targets: ["screen", "window", "element", "browser"] as const,
|
||||
deliveryModes: ["background", "foreground"] as const,
|
||||
observations: ["image", "accessibility", "browser"] as const,
|
||||
features: { recording: false, agentCursor: false, multiDisplay: false },
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function readActionEnum(tool: ComputerTool): string[] {
|
||||
const schema = tool.parameters as { properties?: { action?: { enum?: string[] } } };
|
||||
return schema.properties?.action?.enum ?? [];
|
||||
}
|
||||
|
||||
describe("createComputerTool schema", () => {
|
||||
it("keeps an undeclared node on the exact v1 action list", () => {
|
||||
expect(readActionEnum(createComputerTool())).toEqual([
|
||||
"screenshot",
|
||||
"left_click",
|
||||
"right_click",
|
||||
"middle_click",
|
||||
"double_click",
|
||||
"triple_click",
|
||||
"mouse_move",
|
||||
"left_click_drag",
|
||||
"left_mouse_down",
|
||||
"left_mouse_up",
|
||||
"scroll",
|
||||
"type",
|
||||
"key",
|
||||
"hold_key",
|
||||
"wait",
|
||||
]);
|
||||
});
|
||||
|
||||
it("filters the model schema to a preselected v2 descriptor", () => {
|
||||
const actions: ComputerUseV2ActionName[] = ["screenshot", "list_apps", "get_window_state"];
|
||||
const tool = createComputerTool({ capabilityDescriptor: v2Descriptor(actions) });
|
||||
expect(readActionEnum(tool)).toEqual(actions);
|
||||
});
|
||||
|
||||
it("keeps the v2 guidance provider-neutral and free of host setup instructions", () => {
|
||||
const description = createComputerTool({
|
||||
capabilityDescriptor: v2Descriptor([
|
||||
"screenshot",
|
||||
"left_click",
|
||||
"list_windows",
|
||||
"get_window_state",
|
||||
"set_value",
|
||||
]),
|
||||
}).description;
|
||||
|
||||
expect(description).toContain("Observe first with `get_window_state`");
|
||||
expect(description).toContain('`effect:"confirmed"` > `unverifiable` > `suspected_noop`');
|
||||
expect(description).toContain("never blind-retry a mutation");
|
||||
expect(description).toContain("untrusted input");
|
||||
expect(description).not.toMatch(
|
||||
/cua|peekaboo|\b(?:cli|mcp|daemon|socket|install(?:ation|ing)?)\b|verify_state|start_session|end_session|element_token|snapshot_id|window_id|delivery_mode/iu,
|
||||
);
|
||||
expect(description.length).toBeLessThan(2_400);
|
||||
});
|
||||
|
||||
it("filters guidance to the selected node's advertised capability families", () => {
|
||||
const desktopOnly = createComputerTool({
|
||||
capabilityDescriptor: v2Descriptor(["screenshot", "left_click"], {
|
||||
targets: ["screen"],
|
||||
deliveryModes: ["foreground"],
|
||||
observations: ["image"],
|
||||
}),
|
||||
}).description;
|
||||
expect(desktopOnly).toContain("desktop coordinates from the latest screenshot");
|
||||
expect(desktopOnly).toContain("stale frameId");
|
||||
expect(desktopOnly).not.toMatch(
|
||||
/get_window_state|accessibility|elementRef|window pixels|deliveryMode:"background"|background_unavailable/,
|
||||
);
|
||||
|
||||
const windowBackground = createComputerTool({
|
||||
capabilityDescriptor: v2Descriptor(
|
||||
["left_click", "list_windows", "get_window_state", "set_value"],
|
||||
{
|
||||
targets: ["window", "element"],
|
||||
deliveryModes: ["background"],
|
||||
},
|
||||
),
|
||||
}).description;
|
||||
expect(windowBackground).toContain(
|
||||
"elementRef from the latest observation > window pixels from the latest window image",
|
||||
);
|
||||
expect(windowBackground).toContain('deliveryMode:"background"');
|
||||
expect(windowBackground).toContain("background_occluded");
|
||||
expect(windowBackground).not.toMatch(/desktop coordinates|foreground|frameId/);
|
||||
});
|
||||
|
||||
it("publishes Codex-compatible fixed-size coordinate arrays", () => {
|
||||
const properties = (
|
||||
createComputerTool().parameters as {
|
||||
properties?: Record<string, Record<string, unknown>>;
|
||||
}
|
||||
).properties;
|
||||
|
||||
for (const key of ["coordinate", "startCoordinate"] as const) {
|
||||
const schema = properties?.[key];
|
||||
if (!schema) {
|
||||
throw new Error(`missing ${key} schema`);
|
||||
}
|
||||
expect(schema).toMatchObject({
|
||||
type: "array",
|
||||
items: { type: "integer", minimum: 0 },
|
||||
minItems: 2,
|
||||
maxItems: 2,
|
||||
});
|
||||
expect(Array.isArray(schema.items)).toBe(false);
|
||||
expect(schema).not.toHaveProperty("additionalItems");
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -317,109 +317,6 @@ describe("computer screenshot context binding", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("createComputerTool schema", () => {
|
||||
it("keeps an undeclared node on the exact v1 action list", () => {
|
||||
expect(readActionEnum(createComputerTool())).toEqual([
|
||||
"screenshot",
|
||||
"left_click",
|
||||
"right_click",
|
||||
"middle_click",
|
||||
"double_click",
|
||||
"triple_click",
|
||||
"mouse_move",
|
||||
"left_click_drag",
|
||||
"left_mouse_down",
|
||||
"left_mouse_up",
|
||||
"scroll",
|
||||
"type",
|
||||
"key",
|
||||
"hold_key",
|
||||
"wait",
|
||||
]);
|
||||
});
|
||||
|
||||
it("filters the model schema to a preselected v2 descriptor", () => {
|
||||
const actions: ComputerUseV2ActionName[] = ["screenshot", "list_apps", "get_window_state"];
|
||||
const tool = createComputerTool({ capabilityDescriptor: v2Descriptor(actions) });
|
||||
expect(readActionEnum(tool)).toEqual(actions);
|
||||
});
|
||||
|
||||
it("keeps the v2 guidance provider-neutral and free of host setup instructions", () => {
|
||||
const description = createComputerTool({
|
||||
capabilityDescriptor: v2Descriptor([
|
||||
"screenshot",
|
||||
"left_click",
|
||||
"list_windows",
|
||||
"get_window_state",
|
||||
"set_value",
|
||||
]),
|
||||
}).description;
|
||||
|
||||
expect(description).toContain("Observe first with `get_window_state`");
|
||||
expect(description).toContain('`effect:"confirmed"` > `unverifiable` > `suspected_noop`');
|
||||
expect(description).toContain("never blind-retry a mutation");
|
||||
expect(description).toContain("untrusted input");
|
||||
expect(description).not.toMatch(
|
||||
/cua|peekaboo|\b(?:cli|mcp|daemon|socket|install(?:ation|ing)?)\b|verify_state|start_session|end_session|element_token|snapshot_id|window_id|delivery_mode/iu,
|
||||
);
|
||||
expect(description.length).toBeLessThan(2_400);
|
||||
});
|
||||
|
||||
it("filters guidance to the selected node's advertised capability families", () => {
|
||||
const desktopOnly = createComputerTool({
|
||||
capabilityDescriptor: v2Descriptor(["screenshot", "left_click"], {
|
||||
targets: ["screen"],
|
||||
deliveryModes: ["foreground"],
|
||||
observations: ["image"],
|
||||
}),
|
||||
}).description;
|
||||
expect(desktopOnly).toContain("desktop coordinates from the latest screenshot");
|
||||
expect(desktopOnly).toContain("stale frameId");
|
||||
expect(desktopOnly).not.toMatch(
|
||||
/get_window_state|accessibility|elementRef|window pixels|deliveryMode:"background"|background_unavailable/,
|
||||
);
|
||||
|
||||
const windowBackground = createComputerTool({
|
||||
capabilityDescriptor: v2Descriptor(
|
||||
["left_click", "list_windows", "get_window_state", "set_value"],
|
||||
{
|
||||
targets: ["window", "element"],
|
||||
deliveryModes: ["background"],
|
||||
},
|
||||
),
|
||||
}).description;
|
||||
expect(windowBackground).toContain(
|
||||
"elementRef from the latest observation > window pixels from the latest window image",
|
||||
);
|
||||
expect(windowBackground).toContain('deliveryMode:"background"');
|
||||
expect(windowBackground).toContain("background_occluded");
|
||||
expect(windowBackground).not.toMatch(/desktop coordinates|foreground|frameId/);
|
||||
});
|
||||
|
||||
it("publishes Codex-compatible fixed-size coordinate arrays", () => {
|
||||
const properties = (
|
||||
createComputerTool().parameters as {
|
||||
properties?: Record<string, Record<string, unknown>>;
|
||||
}
|
||||
).properties;
|
||||
|
||||
for (const key of ["coordinate", "startCoordinate"] as const) {
|
||||
const schema = properties?.[key];
|
||||
if (!schema) {
|
||||
throw new Error(`missing ${key} schema`);
|
||||
}
|
||||
expect(schema).toMatchObject({
|
||||
type: "array",
|
||||
items: { type: "integer", minimum: 0 },
|
||||
minItems: 2,
|
||||
maxItems: 2,
|
||||
});
|
||||
expect(Array.isArray(schema.items)).toBe(false);
|
||||
expect(schema).not.toHaveProperty("additionalItems");
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("createComputerTool execution", () => {
|
||||
beforeEach(() => {
|
||||
listNodesMock.mockReset();
|
||||
@@ -584,6 +481,50 @@ describe("createComputerTool execution", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("routes an observation-bound element click without requiring coordinates", async () => {
|
||||
const actions: ComputerUseV2ActionName[] = ["get_window_state", "left_click"];
|
||||
listNodesMock.mockResolvedValue([macComputerNode({ computerUse: v2Descriptor(actions) })]);
|
||||
callGatewayToolMock.mockImplementation(async (_method, _opts, body) => {
|
||||
const request = body as ComputerActBody;
|
||||
if (request.command !== COMPUTER_ACT_COMMAND) {
|
||||
return screenshotPayload();
|
||||
}
|
||||
if (request.params?.action === "get_window_state") {
|
||||
return {
|
||||
payload: {
|
||||
ok: true,
|
||||
observation: {
|
||||
kind: "window",
|
||||
observationId: "observation-1",
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
return { payload: { ok: true, effect: "confirmed" } };
|
||||
});
|
||||
const tool = createVisionComputerTool({ capabilityDescriptor: v2Descriptor(actions) });
|
||||
await tool.execute("observe", { action: "get_window_state", windowRef: "window-1" });
|
||||
|
||||
await expect(
|
||||
tool.execute("click", {
|
||||
action: "left_click",
|
||||
windowRef: "window-1",
|
||||
elementRef: "element-1",
|
||||
observationId: "observation-1",
|
||||
deliveryMode: "background",
|
||||
}),
|
||||
).resolves.toBeDefined();
|
||||
expect(readLastComputerActParams()).toEqual({
|
||||
action: "left_click",
|
||||
screenIndex: 0,
|
||||
refWidth: EFFECTIVE_REF_WIDTH,
|
||||
windowRef: "window-1",
|
||||
elementRef: "element-1",
|
||||
observationId: "observation-1",
|
||||
deliveryMode: "background",
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects recording actions that remain contract-only", async () => {
|
||||
const actions: ComputerUseV2ActionName[] = ["start_recording"];
|
||||
listNodesMock.mockResolvedValue([macComputerNode({ computerUse: v2Descriptor(actions) })]);
|
||||
|
||||
@@ -102,6 +102,14 @@ const COORDINATE_REQUIRED_ACTIONS = new Set<ComputerToolAction>([
|
||||
"left_click_drag",
|
||||
]);
|
||||
|
||||
const ELEMENT_TARGETABLE_CLICK_ACTIONS = new Set<ComputerToolAction>([
|
||||
"left_click",
|
||||
"right_click",
|
||||
"middle_click",
|
||||
"double_click",
|
||||
"triple_click",
|
||||
]);
|
||||
|
||||
// Actions that accept an optional target coordinate (scroll at a point, press
|
||||
// or release the button at a point). Keyboard actions never carry coordinates.
|
||||
const COORDINATE_OPTIONAL_ACTIONS = new Set<ComputerToolAction>([
|
||||
@@ -369,7 +377,11 @@ function buildComputerActParams(params: {
|
||||
wire.screenIndex = params.screenIndex;
|
||||
wire.refWidth = params.refWidth ?? COMPUTER_REF_WIDTH;
|
||||
}
|
||||
if (COORDINATE_REQUIRED_ACTIONS.has(action)) {
|
||||
const elementRef = readToolStringParam(input, "elementRef");
|
||||
if (
|
||||
COORDINATE_REQUIRED_ACTIONS.has(action) &&
|
||||
!(elementRef && ELEMENT_TARGETABLE_CLICK_ACTIONS.has(action))
|
||||
) {
|
||||
const [x, y] = requireCoordinate(input, action);
|
||||
wire.x = x;
|
||||
wire.y = y;
|
||||
|
||||
Reference in New Issue
Block a user