feat(cua-computer): full v2 adapter with background window/element delivery (#123604)

* feat(cua-computer): add full v2 adapter

* chore(cua-computer): drop release-owned changelog edit

extensions/cua-computer/src/commands.test.ts

@@ -1,7 +1,12 @@
 import { describe, expect, it, vi } from "vitest";
 import { createCuaComputerProvider } from "./commands.js";
+import {
+  CUA_DRIVER_CONTRACT_FIXTURES,
+  cuaToolResult,
+} from "./cua-driver-contract.test-fixtures.js";
 import {
   ClickButton,
+  EscalationReason,
   ScrollDirection,
   type CuaDriverSession,
   type CuaToolResult,
@@ -31,6 +36,7 @@ function result(structured: Record<string, unknown>, image = false): CuaToolResu
 }
 
 function driver() {
+  let generation = "execution-1";
   const getDesktopState = vi.fn(async () => result(geometry, true));
   const getScreenSize = vi.fn(async () => result({ width: 100, height: 50, scale_factor: 1 }));
   const click = vi.fn(async () => result({}));
@@ -39,11 +45,22 @@ function driver() {
   const scroll = vi.fn(async () => result({}));
   const typeText = vi.fn(async () => result({}));
   const pressKey = vi.fn(async () => result({}));
+  const callTool = vi.fn<CuaDriverSession["callTool"]>(async () => result({}));
+  const escalateScope = vi.fn(async () => ({
+    session: "openclaw-test",
+    captureScope: 2,
+    effectiveScope: 1,
+    desktopUnlocked: true,
+  }));
   const dispose = vi.fn(async () => {});
   const session: CuaDriverSession = {
-    generation: "execution-1",
+    get generation() {
+      return generation;
+    },
     isAvailable: () => true,
     resetAvailabilityCache: () => {},
+    callTool,
+    escalateScope,
     getDesktopState,
     getScreenSize,
     click,
@@ -62,9 +79,14 @@ function driver() {
     drag,
     moveCursor,
     scroll,
+    callTool,
+    escalateScope,
     dispose,
     typeText,
     pressKey,
+    setGeneration: (value: string) => {
+      generation = value;
+    },
   };
 }
 
@@ -79,7 +101,7 @@ async function execution(session: CuaDriverSession) {
 }
 
 describe("cua-computer provider", () => {
-  it("advertises only its current foreground coordinate capability", () => {
+  it("advertises the implemented Linux v2 capability", () => {
     const { session } = driver();
     const descriptor = createCuaComputerProvider({
       platform: "linux",
@@ -90,7 +112,7 @@ describe("cua-computer provider", () => {
       provider: {
         id: "cua-computer",
         label: "CUA Computer",
-        generation: "cua-computer-coordinate-v1",
+        generation: "cua-computer-v2:execution-1",
       },
       actions: [
         "screenshot",
@@ -106,16 +128,35 @@ describe("cua-computer provider", () => {
         "scroll",
         "type",
         "key",
-        "hold_key",
+        "list_apps",
-        "wait",
+        "list_windows",
+        "get_accessibility_tree",
+        "get_cursor_position",
+        "get_window_state",
+        "launch_app",
+        "kill_app",
+        "bring_to_front",
+        "set_value",
+        "zoom",
+        "escalate_scope",
+        "invoke_menu",
       ],
-      targets: ["screen"],
+      targets: ["screen", "window", "element"],
-      deliveryModes: ["foreground"],
+      deliveryModes: ["background", "foreground"],
-      observations: ["image"],
+      observations: ["image", "accessibility"],
       features: { recording: false, agentCursor: false, multiDisplay: false },
     });
   });
 
+  it("omits Linux-only held-button actions on Windows", () => {
+    const { session } = driver();
+    const actions = createCuaComputerProvider({ platform: "win32", driver: session }).capabilities()
+      .actions;
+    expect(actions).not.toContain("left_mouse_down");
+    expect(actions).not.toContain("left_mouse_up");
+    expect(actions).toContain("get_window_state");
+  });
+
   it("uses one typed session for snapshot and frame-authorized click", async () => {
     const { session, getDesktopState, getScreenSize, click } = driver();
     const computer = await execution(session);
@@ -279,4 +320,359 @@ describe("cua-computer provider", () => {
     await computer.snapshot('{"format":"png","maxWidth":100}', signal);
     expect(getDesktopState).toHaveBeenCalledWith(signal);
   });
+
+  it("mints opaque window and element references and maps background evidence", async () => {
+    const { session, callTool } = driver();
+    callTool.mockImplementation(async (name) => {
+      switch (name) {
+        case "list_windows":
+          return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.listWindows);
+        case "get_window_state":
+          return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.windowState, { image: true });
+        case "click":
+          return cuaToolResult(
+            {},
+            {
+              action:
+                CUA_DRIVER_CONTRACT_FIXTURES.confirmedBackgroundAction as unknown as CuaToolResult["action"],
+            },
+          );
+        default:
+          return cuaToolResult({});
+      }
+    });
+    const computer = await execution(session);
+    const listed = JSON.parse(await computer.act('{"action":"list_windows"}')) as {
+      details: { windows: Array<{ windowRef: string }> };
+    };
+    const windowRef = listed.details.windows[0]!.windowRef;
+    expect(windowRef).toMatch(/^cua:v2:window:/);
+
+    const observed = JSON.parse(
+      await computer.act(JSON.stringify({ action: "get_window_state", windowRef })),
+    ) as {
+      observation: {
+        observationId: string;
+        elements: Array<{ elementRef: string }>;
+      };
+    };
+    const { observationId } = observed.observation;
+    const elementRef = observed.observation.elements[0]!.elementRef;
+    expect(observationId).toMatch(/^cua:v2:observation:/);
+    expect(elementRef).toMatch(/^cua:v2:element:/);
+
+    const clicked = JSON.parse(
+      await computer.act(
+        JSON.stringify({
+          action: "left_click",
+          windowRef,
+          elementRef,
+          observationId,
+          deliveryMode: "background",
+        }),
+      ),
+    ) as { effect: string; details: Record<string, unknown> };
+    expect(clicked).toMatchObject({
+      ok: true,
+      effect: "confirmed",
+      details: {
+        route: "accessibility",
+        deliveryMode: "background",
+        deliveredCount: 1,
+        evidence: ["value_readback"],
+      },
+    });
+    expect(callTool).toHaveBeenLastCalledWith(
+      "click",
+      {
+        pid: 4242,
+        window_id: 99,
+        element_token: "native-element-token-7",
+        button: "left",
+        count: 1,
+        delivery_mode: "background",
+      },
+      undefined,
+    );
+  });
+
+  it("maps window pixels, app lifecycle, menu, zoom, and escalation tools", async () => {
+    const { session, callTool, escalateScope } = driver();
+    callTool.mockImplementation(async (name) => {
+      switch (name) {
+        case "list_apps":
+          return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.listApps);
+        case "list_windows":
+          return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.listWindows);
+        case "get_window_state":
+          return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.windowState, { image: true });
+        case "zoom":
+          return cuaToolResult({ screenshot_width: 300, screenshot_height: 200 }, { image: true });
+        default:
+          return cuaToolResult(
+            {},
+            {
+              action:
+                CUA_DRIVER_CONTRACT_FIXTURES.suspectedNoopAction as unknown as CuaToolResult["action"],
+            },
+          );
+      }
+    });
+    const computer = await execution(session);
+    const apps = JSON.parse(await computer.act('{"action":"list_apps"}')) as {
+      details: { apps: Array<{ app: string }> };
+    };
+    const app = apps.details.apps[0]!.app;
+    const windows = JSON.parse(await computer.act('{"action":"list_windows"}')) as {
+      details: { windows: Array<{ windowRef: string }> };
+    };
+    const windowRef = windows.details.windows[0]!.windowRef;
+    const observed = JSON.parse(
+      await computer.act(JSON.stringify({ action: "get_window_state", windowRef })),
+    ) as { observation: { observationId: string } };
+
+    await computer.act(JSON.stringify({ action: "launch_app", app }));
+    await computer.act(JSON.stringify({ action: "kill_app", app }));
+    await computer.act(
+      JSON.stringify({ action: "invoke_menu", windowRef, path: ["File", "Save"] }),
+    );
+    const zoomed = JSON.parse(
+      await computer.act(
+        JSON.stringify({
+          action: "zoom",
+          windowRef,
+          observationId: observed.observation.observationId,
+          x1: 0,
+          y1: 0,
+          x2: 100,
+          y2: 100,
+        }),
+      ),
+    ) as { observation: { observationId: string } };
+    expect(zoomed.observation.observationId).not.toBe(observed.observation.observationId);
+    await computer.act(
+      JSON.stringify({ action: "escalate_scope", reason: "background_delivery_failed" }),
+    );
+
+    expect(callTool).toHaveBeenCalledWith(
+      "launch_app",
+      { launch_path: "/usr/bin/editor" },
+      undefined,
+    );
+    expect(callTool).toHaveBeenCalledWith("kill_app", { pid: 4242 }, undefined);
+    expect(callTool).toHaveBeenCalledWith(
+      "invoke_menu",
+      { pid: 4242, window_id: 99, path: ["File", "Save"] },
+      undefined,
+    );
+    expect(escalateScope).toHaveBeenCalledWith(
+      EscalationReason.BackgroundDeliveryFailed,
+      undefined,
+    );
+  });
+
+  it("maps the complete Linux window pointer and keyboard family", async () => {
+    const { session, callTool } = driver();
+    callTool.mockImplementation(async (name) => {
+      if (name === "list_windows") {
+        return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.listWindows);
+      }
+      if (name === "get_window_state") {
+        return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.windowState, { image: true });
+      }
+      return cuaToolResult(
+        {},
+        {
+          action:
+            CUA_DRIVER_CONTRACT_FIXTURES.confirmedBackgroundAction as unknown as CuaToolResult["action"],
+        },
+      );
+    });
+    const computer = await execution(session);
+    const listed = JSON.parse(await computer.act('{"action":"list_windows"}')) as {
+      details: { windows: Array<{ windowRef: string }> };
+    };
+    const windowRef = listed.details.windows[0]!.windowRef;
+    const observed = JSON.parse(
+      await computer.act(JSON.stringify({ action: "get_window_state", windowRef })),
+    ) as {
+      observation: { observationId: string; elements: Array<{ elementRef: string }> };
+    };
+    const observationId = observed.observation.observationId;
+    const elementRef = observed.observation.elements[0]!.elementRef;
+    const pixelTarget = { windowRef, observationId, x: 20, y: 30 };
+    const cases = [
+      ["right_click", "click", { button: "right", count: 1 }],
+      ["middle_click", "click", { button: "middle", count: 1 }],
+      ["double_click", "click", { button: "left", count: 2 }],
+      ["triple_click", "click", { button: "left", count: 3 }],
+      ["left_click_drag", "drag", { from_x: 10, from_y: 15, to_x: 20, to_y: 30, duration_ms: 250 }],
+      ["left_mouse_down", "mouse_button_down", { x: 20, y: 30, button: "left" }],
+      ["left_mouse_up", "mouse_button_up", { x: 20, y: 30 }],
+      ["scroll", "scroll", { direction: "down", by: "line", amount: 4 }],
+      ["type", "type_text", { text: "hello", element_token: "native-element-token-7" }],
+      ["key", "press_key", { key: "enter", modifiers: ["ctrl"] }],
+    ] as const;
+
+    for (const [action, tool, expected] of cases) {
+      const actionInput: Record<string, unknown> = {
+        action,
+        ...pixelTarget,
+        deliveryMode: action.startsWith("left_mouse_") ? "background" : "foreground",
+      };
+      if (action === "left_click_drag") {
+        actionInput.fromX = 10;
+        actionInput.fromY = 15;
+        actionInput.durationMs = 250;
+      } else if (action === "scroll") {
+        actionInput.scrollDirection = "down";
+        actionInput.scrollAmount = 4;
+      } else if (action === "type") {
+        actionInput.elementRef = elementRef;
+        actionInput.text = "hello";
+        delete actionInput.x;
+        delete actionInput.y;
+      } else if (action === "key") {
+        actionInput.keys = "ctrl+enter";
+        delete actionInput.x;
+        delete actionInput.y;
+      }
+      await computer.act(JSON.stringify(actionInput));
+      expect(callTool).toHaveBeenCalledWith(
+        tool,
+        expect.objectContaining({ pid: 4242, window_id: 99, ...expected }),
+        undefined,
+      );
+    }
+  });
+
+  it("maps remaining discovery, window lifecycle, and semantic actions", async () => {
+    const { session, callTool } = driver();
+    callTool.mockImplementation(async (name) => {
+      if (name === "list_windows") {
+        return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.listWindows);
+      }
+      if (name === "get_window_state") {
+        return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.windowState, { image: true });
+      }
+      if (name === "get_accessibility_tree") {
+        return cuaToolResult({
+          processes: [{ pid: 4242, name: "Editor" }],
+          windows: CUA_DRIVER_CONTRACT_FIXTURES.listWindows.windows,
+        });
+      }
+      if (name === "get_cursor_position") {
+        return cuaToolResult({ x: 11, y: 12, source: "x11" });
+      }
+      return cuaToolResult(
+        {},
+        {
+          action:
+            CUA_DRIVER_CONTRACT_FIXTURES.confirmedBackgroundAction as unknown as CuaToolResult["action"],
+        },
+      );
+    });
+    const computer = await execution(session);
+    const tree = JSON.parse(await computer.act('{"action":"get_accessibility_tree"}')) as {
+      details: { windows: unknown[]; processes: unknown[] };
+    };
+    expect(tree.details.windows).toHaveLength(1);
+    expect(tree.details.processes).toHaveLength(1);
+    await expect(computer.act('{"action":"get_cursor_position"}')).resolves.toContain('"x":11');
+
+    const listed = JSON.parse(await computer.act('{"action":"list_windows"}')) as {
+      details: { windows: Array<{ windowRef: string }> };
+    };
+    const windowRef = listed.details.windows[0]!.windowRef;
+    const observed = JSON.parse(
+      await computer.act(JSON.stringify({ action: "get_window_state", windowRef })),
+    ) as {
+      observation: { observationId: string; elements: Array<{ elementRef: string }> };
+    };
+    await computer.act(JSON.stringify({ action: "bring_to_front", windowRef }));
+    await computer.act(
+      JSON.stringify({
+        action: "set_value",
+        windowRef,
+        observationId: observed.observation.observationId,
+        elementRef: observed.observation.elements[0]!.elementRef,
+        value: "new",
+        deliveryMode: "background",
+      }),
+    );
+    expect(callTool).toHaveBeenCalledWith(
+      "bring_to_front",
+      { pid: 4242, window_id: 99 },
+      undefined,
+    );
+    expect(callTool).toHaveBeenCalledWith(
+      "set_value",
+      {
+        pid: 4242,
+        window_id: 99,
+        element_token: "native-element-token-7",
+        value: "new",
+      },
+      undefined,
+    );
+  });
+
+  it("maps window delivery refusals to the closed computer error prefix", async () => {
+    const { session, callTool } = driver();
+    callTool.mockImplementation(async (name) => {
+      if (name === "list_windows") {
+        return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.listWindows);
+      }
+      if (name === "get_window_state") {
+        return cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.windowState, { image: true });
+      }
+      return cuaToolResult(
+        { code: "background_occluded" },
+        {
+          isError: true,
+          errorCode: "background_occluded",
+          text: "target is occluded",
+        },
+      );
+    });
+    const computer = await execution(session);
+    const listed = JSON.parse(await computer.act('{"action":"list_windows"}')) as {
+      details: { windows: Array<{ windowRef: string }> };
+    };
+    const windowRef = listed.details.windows[0]!.windowRef;
+    const observed = JSON.parse(
+      await computer.act(JSON.stringify({ action: "get_window_state", windowRef })),
+    ) as { observation: { observationId: string } };
+    await expect(
+      computer.act(
+        JSON.stringify({
+          action: "left_click",
+          windowRef,
+          observationId: observed.observation.observationId,
+          x: 10,
+          y: 20,
+        }),
+      ),
+    ).rejects.toThrow("COMPUTER_REFUSED_background_occluded");
+  });
+
+  it("invalidates observation references when the driver generation rotates", async () => {
+    const { session, callTool, setGeneration } = driver();
+    callTool.mockImplementation(async (name) =>
+      name === "list_windows"
+        ? cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.listWindows)
+        : cuaToolResult(CUA_DRIVER_CONTRACT_FIXTURES.windowState, { image: true }),
+    );
+    const computer = await execution(session);
+    const listed = JSON.parse(await computer.act('{"action":"list_windows"}')) as {
+      details: { windows: Array<{ windowRef: string }> };
+    };
+    const windowRef = listed.details.windows[0]!.windowRef;
+    setGeneration("execution-2");
+
+    await expect(
+      computer.act(JSON.stringify({ action: "get_window_state", windowRef })),
+    ).rejects.toThrow("COMPUTER_STALE_OBSERVATION");
+    expect(callTool).toHaveBeenCalledTimes(1);
+  });
 });

extensions/cua-computer/src/commands.ts

@@ -19,7 +19,9 @@ import {
   type CuaDriverSession,
   type CuaToolResult,
 } from "./driver-client.js";
+import { platformActions } from "./driver-result.js";
 import {
+  adoptGeneration,
   issueFrame,
   verifyFrame,
   verifyReferenceWidth,
@@ -28,26 +30,10 @@ import {
   type CuaLastFrame,
   type CuaScreenSize,
 } from "./frame.js";
+import { handleV2Act, type CuaComputerActParams } from "./v2-actions.js";
 
 const AVAILABILITY_POLL_MS = 5_000;
-const CUA_COORDINATE_ACTION_NAMES = COMPUTER_USE_V2_ACTION_NAMES.slice(0, 15);
 const CUA_WIRE_ACTION_NAMES = COMPUTER_USE_V2_ACTION_NAMES.slice(1, 14);
-type CuaComputerActParams = {
-  action: ComputerActParams["action"];
-  displayFrameId?: string;
-  x?: number;
-  y?: number;
-  fromX?: number;
-  fromY?: number;
-  text?: string;
-  keys?: string;
-  modifiers?: string;
-  scrollDirection?: "up" | "down" | "left" | "right";
-  scrollAmount?: number;
-  durationMs?: number;
-  screenIndex?: number;
-  refWidth?: number;
-};
 // Rastermill enforces inputPixels before resizing, so this must clear the native
 // capture, not the delivered frame. 8K (7680x4320 = ~33.2M) is a valid primary
 // display; budget above it so full-resolution snapshots reach the downscaler.
@@ -244,7 +230,7 @@ async function currentFrame(
   return frame;
 }
 
-async function handleAct(
+async function handleDesktopAct(
   driver: CuaDriverSession,
   frameState: CuaFrameState,
   params: ComputerActParams,
@@ -441,12 +427,14 @@ export function createCuaComputerProvider(
       provider: {
         id: "cua-computer",
         label: "CUA Computer",
-        generation: "cua-computer-coordinate-v1",
+        generation: isSupportedPlatform
+          ? `cua-computer-v2:${driver().generation}`
+          : "cua-computer-v2:unsupported",
       },
-      actions: CUA_COORDINATE_ACTION_NAMES,
+      actions: platformActions(platform),
-      targets: ["screen"],
+      targets: ["screen", "window", "element"],
-      deliveryModes: ["foreground"],
+      deliveryModes: ["background", "foreground"],
-      observations: ["image"],
+      observations: ["image", "accessibility"],
       features: { recording: false, agentCursor: false, multiDisplay: false },
     }),
     isAvailable,
@@ -468,7 +456,7 @@ export function createCuaComputerProvider(
     },
     openExecution: async () => {
       const queue = new PromiseQueue();
-      const frameState: CuaFrameState = { generation: "uninitialized" };
+      const frameState: CuaFrameState = { generation: driver().generation };
       return {
         snapshot: async (paramsJSON, signal) =>
           await queue.run(async () => {
@@ -511,7 +499,7 @@ export function createCuaComputerProvider(
               width = result.width;
               height = result.height;
             }
-            frameState.generation = driver().generation;
+            adoptGeneration(frameState, driver().generation);
             const displayFrameId = issueFrame(frameState, geometry, { width, height });
             return JSON.stringify({
               format,
@@ -529,10 +517,12 @@ export function createCuaComputerProvider(
                 "COMPUTER_DRIVER_UNAVAILABLE: cua-computer supports Windows and Linux",
               );
             }
-            return await handleAct(
+            return await handleV2Act(
+              platform,
               driver(),
               frameState,
               parseComputerActParamsJSON(paramsJSON),
+              handleDesktopAct,
               signal,
             );
           }),

extensions/cua-computer/src/cua-driver-contract.test-fixtures.ts

@@ -0,0 +1,88 @@
+import type { CuaToolResult } from "./driver-client.js";
+
+export const CUA_DRIVER_CONTRACT_FIXTURES = {
+  listApps: {
+    apps: [
+      {
+        pid: 4242,
+        bundle_id: "org.example.Editor",
+        name: "Editor",
+        running: true,
+        active: false,
+        kind: "desktop",
+        launch_path: "/usr/bin/editor",
+        last_used: "2026-08-14T00:00:00Z",
+      },
+    ],
+  },
+  listWindows: {
+    windows: [
+      {
+        window_id: 99,
+        pid: 4242,
+        app_name: "Editor",
+        title: "Notes",
+        bounds: { x: 40, y: 50, width: 800, height: 600 },
+        is_on_screen: true,
+        minimized: false,
+        z_index: 2,
+      },
+    ],
+  },
+  windowState: {
+    window_id: 99,
+    pid: 4242,
+    snapshot_id: "native-snapshot-1",
+    total_element_count: 1,
+    returned_element_count: 1,
+    screenshot_width: 800,
+    screenshot_height: 600,
+    screenshot_mime_type: "image/png",
+    elements: [
+      {
+        element_index: 7,
+        element_token: "native-element-token-7",
+        role: "text field",
+        label: "Body",
+        value: "old",
+        frame: { x: 80, y: 100, w: 400, h: 240 },
+      },
+    ],
+  },
+  confirmedBackgroundAction: {
+    effect: 0,
+    route: 0,
+    delivery: { mode: 0, deliveredCount: 1 },
+    evidence: [{ kind: 0 }],
+  },
+  suspectedNoopAction: {
+    effect: 3,
+    route: 1,
+    delivery: { mode: 0 },
+    escalation: { target: 1, reason: 3 },
+  },
+} as const;
+
+export function cuaToolResult(
+  structured: Record<string, unknown>,
+  options: {
+    action?: CuaToolResult["action"];
+    image?: boolean;
+    isError?: boolean;
+    errorCode?: string;
+    text?: string;
+  } = {},
+): CuaToolResult {
+  return {
+    text: options.text ?? "ok",
+    images: options.image
+      ? [{ mimeType: "image/png", dataBase64: Buffer.from("png").toString("base64") }]
+      : [],
+    structuredJson: JSON.stringify(structured),
+    isError: options.isError ?? false,
+    ...(options.errorCode ? { errorCode: options.errorCode } : {}),
+    ...(options.action ? { action: options.action } : {}),
+    degraded: false,
+    rawJson: "{}",
+  };
+}

extensions/cua-computer/src/driver-client.test.ts

@@ -2,10 +2,17 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
 
 const mocks = vi.hoisted(() => ({
   close: vi.fn(),
+  callTool: vi.fn(async () => ({})),
   create: vi.fn(),
   createConfigured: vi.fn(),
   createTrustedSession: vi.fn(),
   endSession: vi.fn(async () => ({})),
+  escalateSession: vi.fn(async () => ({
+    session: "openclaw-test",
+    captureScope: "desktop",
+    effectiveScope: "desktop",
+    desktopUnlocked: true,
+  })),
   getDesktopState: vi.fn(async () => ({})),
   isAvailable: vi.fn(() => true),
   startSession: vi.fn(async () => ({})),
@@ -13,10 +20,11 @@ const mocks = vi.hoisted(() => ({
 }));
 
 const sdk = {
-  CaptureScope: { Desktop: "desktop" },
+  CaptureScope: { Window: "window", Desktop: "desktop" },
   ClickButton: { Left: 0, Right: 1, Middle: 2 },
   CuaDriver: { create: mocks.create, createConfigured: mocks.createConfigured },
   DesktopScope: { Desktop: 0 },
+  EscalationReason: { Other: "other" },
   ScrollBy: { Line: 0 },
   ScrollDirection: { Up: 0, Down: 1, Left: 2, Right: 3 },
   SessionPermissionMode: { Unrestricted: "unrestricted" },
@@ -42,7 +50,9 @@ describe("CUA Driver direct session", () => {
     });
     mocks.createTrustedSession.mockReturnValue({
       close: mocks.close,
+      callTool: mocks.callTool,
       endSession: mocks.endSession,
+      escalateSession: mocks.escalateSession,
       getDesktopState: mocks.getDesktopState,
       startSession: mocks.startSession,
     });
@@ -109,6 +119,33 @@ describe("CUA Driver direct session", () => {
     expect(mocks.endSession).toHaveBeenCalledWith({ session: sessionOptions.publicSession });
   });
 
+  it("starts window-scoped generic tools and widens only for an explicit desktop call", async () => {
+    const driver = createCuaDriver({ loadSdk: () => sdk as never });
+
+    await driver.callTool("list_windows", {});
+    const sessionOptions = mocks.createTrustedSession.mock.calls[0]?.[1];
+    expect(mocks.startSession).toHaveBeenCalledWith(
+      { session: sessionOptions.publicSession, captureScope: "window" },
+      undefined,
+    );
+    expect(mocks.callTool).toHaveBeenCalledWith(
+      "list_windows",
+      JSON.stringify({ session: sessionOptions.publicSession }),
+      undefined,
+    );
+
+    await driver.getDesktopState();
+    expect(mocks.escalateSession).toHaveBeenCalledWith(
+      {
+        session: sessionOptions.publicSession,
+        reason: "other",
+        detail: "explicit desktop-scope OpenClaw action",
+      },
+      undefined,
+    );
+    await driver.dispose();
+  });
+
   it("keeps a missing native desktop library behind command availability", async () => {
     const loadSdk = vi.fn(() => {
       throw new Error("libX11.so.6: cannot open shared object file");

extensions/cua-computer/src/driver-client.ts

@@ -1,14 +1,18 @@
 import { randomUUID } from "node:crypto";
 
 type DriverClickButton = import("@trycua/cua-driver").ClickButton;
+type DriverCaptureScope = import("@trycua/cua-driver").CaptureScope;
+type DriverEscalationReason = import("@trycua/cua-driver").EscalationReason;
 type CuaDriverLike = import("@trycua/cua-driver").CuaDriverLike;
 type CuaDriverSessionLike = import("@trycua/cua-driver").CuaDriverSessionLike;
 type DriverScrollDirection = import("@trycua/cua-driver").ScrollDirection;
+type CuaSessionState = import("@trycua/cua-driver").SessionStateOutput;
 type CuaDriverSdk = Pick<
   typeof import("@trycua/cua-driver"),
   | "CaptureScope"
   | "CuaDriver"
   | "DesktopScope"
+  | "EscalationReason"
   | "ScrollBy"
   | "SessionPermissionMode"
   | "createTrustedSession"
@@ -16,6 +20,15 @@ type CuaDriverSdk = Pick<
 
 export type CuaToolResult = import("@trycua/cua-driver").ToolResult;
 
+export const EscalationReason = {
+  AxTreePixelMismatch: 0 as DriverEscalationReason,
+  BackgroundDeliveryFailed: 1 as DriverEscalationReason,
+  ForegroundIneffective: 2 as DriverEscalationReason,
+  NoWindowTarget: 3 as DriverEscalationReason,
+  Other: 4 as DriverEscalationReason,
+} as const;
+export type EscalationReason = (typeof EscalationReason)[keyof typeof EscalationReason];
+
 // These numeric values are part of the pinned 0.19.3 SDK contract. Keeping
 // them local avoids loading the native library while OpenClaw is only
 // registering the bundled plugin.
@@ -38,6 +51,12 @@ export interface CuaDriverSession {
   readonly generation: string;
   isAvailable(): boolean;
   resetAvailabilityCache(): void;
+  callTool(
+    name: string,
+    args: Record<string, unknown>,
+    signal?: AbortSignal,
+  ): Promise<CuaToolResult>;
+  escalateScope(reason: EscalationReason, signal?: AbortSignal): Promise<CuaSessionState>;
   getDesktopState(signal?: AbortSignal): Promise<CuaToolResult>;
   getScreenSize(signal?: AbortSignal): Promise<CuaToolResult>;
   click(
@@ -71,6 +90,8 @@ class DirectCuaDriverSession implements CuaDriverSession {
   private readonly session: CuaDriverSessionLike;
   private readonly publicSession = `openclaw-${randomUUID()}`;
   private startPromise: Promise<void> | undefined;
+  private desktopEscalationPromise: Promise<void> | undefined;
+  private captureScope: DriverCaptureScope | undefined;
   private started = false;
   private disposed = false;
 
@@ -99,16 +120,17 @@ class DirectCuaDriverSession implements CuaDriverSession {
     });
   }
 
-  private async ensureStarted(signal?: AbortSignal): Promise<void> {
+  private async ensureStarted(
+    captureScope: DriverCaptureScope,
+    signal?: AbortSignal,
+  ): Promise<void> {
     if (this.disposed) {
       throw new Error("COMPUTER_DRIVER_UNAVAILABLE: cua-computer is stopping");
     }
     if (!this.startPromise) {
+      this.captureScope = captureScope;
       const start = this.session
-        .startSession(
+        .startSession({ session: this.publicSession, captureScope }, asyncOptions(signal))
-          { session: this.publicSession, captureScope: this.sdk.CaptureScope.Desktop },
-          asyncOptions(signal),
-        )
         .then(() => {
           this.started = true;
         });
@@ -124,13 +146,38 @@ class DirectCuaDriverSession implements CuaDriverSession {
       return;
     }
     await this.startPromise;
+    if (
+      captureScope === this.sdk.CaptureScope.Desktop &&
+      this.captureScope !== this.sdk.CaptureScope.Desktop
+    ) {
+      await this.ensureDesktopScope(signal);
+    }
+  }
+
+  private async ensureDesktopScope(signal?: AbortSignal): Promise<void> {
+    if (!this.desktopEscalationPromise) {
+      this.desktopEscalationPromise = this.session
+        .escalateSession(
+          {
+            session: this.publicSession,
+            reason: this.sdk.EscalationReason.Other,
+            detail: "explicit desktop-scope OpenClaw action",
+          },
+          asyncOptions(signal),
+        )
+        .then(() => {
+          this.captureScope = this.sdk.CaptureScope.Desktop;
+        });
+    }
+    await this.desktopEscalationPromise;
   }
 
   private async invoke<T>(
+    captureScope: DriverCaptureScope,
     signal: AbortSignal | undefined,
     operation: () => Promise<T>,
   ): Promise<T> {
-    await this.ensureStarted(signal);
+    await this.ensureStarted(captureScope, signal);
     return await operation();
   }
 
@@ -138,17 +185,39 @@ class DirectCuaDriverSession implements CuaDriverSession {
     return !this.disposed && this.runtime.isAvailable();
   }
   resetAvailabilityCache(): void {}
+  async callTool(name: string, args: Record<string, unknown>, signal?: AbortSignal) {
+    return await this.invoke(this.sdk.CaptureScope.Window, signal, () =>
+      this.session.callTool(
+        name,
+        JSON.stringify({ ...args, session: this.publicSession }),
+        asyncOptions(signal),
+      ),
+    );
+  }
+  async escalateScope(reason: EscalationReason, signal?: AbortSignal) {
+    await this.ensureStarted(this.sdk.CaptureScope.Window, signal);
+    const state = await this.session.escalateSession(
+      { session: this.publicSession, reason },
+      asyncOptions(signal),
+    );
+    this.captureScope = this.sdk.CaptureScope.Desktop;
+    return state;
+  }
   async getDesktopState(signal?: AbortSignal) {
-    return await this.invoke(signal, () => this.session.getDesktopState({}, asyncOptions(signal)));
+    return await this.invoke(this.sdk.CaptureScope.Desktop, signal, () =>
+      this.session.getDesktopState({}, asyncOptions(signal)),
+    );
   }
   async getScreenSize(signal?: AbortSignal) {
-    return await this.invoke(signal, () => this.session.getScreenSize({}, asyncOptions(signal)));
+    return await this.invoke(this.sdk.CaptureScope.Desktop, signal, () =>
+      this.session.getScreenSize({}, asyncOptions(signal)),
+    );
   }
   async click(
     input: { x: number; y: number; button: ClickButton; count: number },
     signal?: AbortSignal,
   ) {
-    return await this.invoke(signal, () =>
+    return await this.invoke(this.sdk.CaptureScope.Desktop, signal, () =>
       this.session.click({ ...input, scope: this.sdk.DesktopScope.Desktop }, asyncOptions(signal)),
     );
   }
@@ -156,12 +225,12 @@ class DirectCuaDriverSession implements CuaDriverSession {
     input: { fromX: number; fromY: number; toX: number; toY: number; durationMs?: bigint },
     signal?: AbortSignal,
   ) {
-    return await this.invoke(signal, () =>
+    return await this.invoke(this.sdk.CaptureScope.Desktop, signal, () =>
       this.session.drag({ ...input, scope: this.sdk.DesktopScope.Desktop }, asyncOptions(signal)),
     );
   }
   async moveCursor(input: { x: number; y: number }, signal?: AbortSignal) {
-    return await this.invoke(signal, () =>
+    return await this.invoke(this.sdk.CaptureScope.Desktop, signal, () =>
       this.session.moveCursor(
         { ...input, scope: this.sdk.DesktopScope.Desktop },
         asyncOptions(signal),
@@ -172,7 +241,7 @@ class DirectCuaDriverSession implements CuaDriverSession {
     input: { x: number; y: number; direction: ScrollDirection; amount: bigint },
     signal?: AbortSignal,
   ) {
-    return await this.invoke(signal, () =>
+    return await this.invoke(this.sdk.CaptureScope.Desktop, signal, () =>
       this.session.scroll(
         {
           ...input,
@@ -184,12 +253,12 @@ class DirectCuaDriverSession implements CuaDriverSession {
     );
   }
   async typeText(text: string, signal?: AbortSignal) {
-    return await this.invoke(signal, () =>
+    return await this.invoke(this.sdk.CaptureScope.Desktop, signal, () =>
       this.session.typeText({ text, scope: this.sdk.DesktopScope.Desktop }, asyncOptions(signal)),
     );
   }
   async pressKey(input: { key: string; modifiers: string[] }, signal?: AbortSignal) {
-    return await this.invoke(signal, () =>
+    return await this.invoke(this.sdk.CaptureScope.Desktop, signal, () =>
       this.session.pressKey(
         { ...input, scope: this.sdk.DesktopScope.Desktop },
         asyncOptions(signal),
@@ -344,6 +413,12 @@ class LazyCuaDriverSession implements CuaDriverSession {
   async getDesktopState(signal?: AbortSignal) {
     return await (await this.requireRuntime()).getDesktopState(signal);
   }
+  async callTool(name: string, args: Record<string, unknown>, signal?: AbortSignal) {
+    return await (await this.requireRuntime()).callTool(name, args, signal);
+  }
+  async escalateScope(reason: EscalationReason, signal?: AbortSignal) {
+    return await (await this.requireRuntime()).escalateScope(reason, signal);
+  }
   async getScreenSize(signal?: AbortSignal) {
     return await (await this.requireRuntime()).getScreenSize(signal);
   }

extensions/cua-computer/src/driver-result.ts

@@ -0,0 +1,364 @@
+import {
+  COMPUTER_USE_V2_ACTION_NAMES,
+  type ComputerActResult,
+  type ComputerUseV2ActionName,
+} from "openclaw/plugin-sdk/computer-use";
+import { canonicalizeBase64 } from "openclaw/plugin-sdk/media-runtime";
+import { z } from "zod";
+import type { CuaDriverSession, CuaToolResult } from "./driver-client.js";
+import {
+  adoptGeneration,
+  issueAppRef,
+  issueElementRef,
+  issueObservation,
+  issueWindowRef,
+  type CuaFrameState,
+} from "./frame.js";
+
+const CUA_WIRE_ACTION_NAMES = COMPUTER_USE_V2_ACTION_NAMES.slice(1, 14);
+const CUA_COMMON_ACTION_NAMES = [
+  "screenshot",
+  ...CUA_WIRE_ACTION_NAMES.filter((action) => action !== "hold_key"),
+  "list_apps",
+  "list_windows",
+  "get_accessibility_tree",
+  "get_cursor_position",
+  "get_window_state",
+  "launch_app",
+  "kill_app",
+  "bring_to_front",
+  "set_value",
+  "zoom",
+  "escalate_scope",
+  "invoke_menu",
+] as const;
+
+const NativeAppSchema = z.object({
+  pid: z.number().int().nonnegative().nullable().optional(),
+  bundle_id: z.string().nullable().optional(),
+  name: z.string().min(1),
+  running: z.boolean().nullable().optional(),
+  active: z.boolean().optional(),
+  kind: z.string().nullable().optional(),
+  launch_path: z.string().nullable().optional(),
+  last_used: z.string().nullable().optional(),
+});
+const NativeBoundsSchema = z.object({
+  x: z.number(),
+  y: z.number(),
+  width: z.number().nonnegative(),
+  height: z.number().nonnegative(),
+});
+const NativeWindowSchema = z.object({
+  window_id: z.number().int().nonnegative(),
+  pid: z.number().int().positive().nullable().optional(),
+  app_name: z.string().optional(),
+  title: z.string().optional(),
+  bounds: NativeBoundsSchema,
+  is_on_screen: z.boolean().optional(),
+  minimized: z.boolean().optional(),
+  z_index: z.number().int().nullable().optional(),
+});
+const NativeElementSchema = z.object({
+  element_index: z.number().int().nonnegative(),
+  element_token: z.string().min(1).optional(),
+  role: z.string().optional(),
+  label: z.string().optional(),
+  value: z.string().optional(),
+  frame: z
+    .object({
+      x: z.number(),
+      y: z.number(),
+      w: z.number().nonnegative(),
+      h: z.number().nonnegative(),
+    })
+    .optional(),
+});
+const MAX_DISCOVERY_ITEMS = 500;
+const PARTIAL_EFFECT = 1 as import("@trycua/cua-driver").ActionEffect;
+const VALUE_READBACK_EVIDENCE = 0 as import("@trycua/cua-driver").ActionEvidenceKind;
+
+export function platformActions(platform: NodeJS.Platform): ComputerUseV2ActionName[] {
+  return CUA_COMMON_ACTION_NAMES.filter(
+    (action) =>
+      platform === "linux" || (action !== "left_mouse_down" && action !== "left_mouse_up"),
+  ) as ComputerUseV2ActionName[];
+}
+
+function boundedItems<T>(items: T[]): { items: T[]; truncated: number } {
+  return {
+    items: items.slice(0, MAX_DISCOVERY_ITEMS),
+    truncated: Math.max(0, items.length - MAX_DISCOVERY_ITEMS),
+  };
+}
+
+function driverEffect(result: CuaToolResult): ComputerActResult["effect"] | undefined {
+  switch (Number(result.action?.effect)) {
+    case 0:
+      return "confirmed";
+    case 1:
+    case 2:
+      return "unverifiable";
+    case 3:
+      return "suspected_noop";
+    case 4:
+      throw new Error("COMPUTER_REFUSED_action_refused: CUA Driver refused the action");
+    default:
+      return undefined;
+  }
+}
+
+function driverEscalation(result: CuaToolResult): ComputerActResult["escalation"] | undefined {
+  const escalation = result.action?.escalation;
+  if (!escalation) {
+    return undefined;
+  }
+  const recommended = {
+    0: "window-pixel",
+    1: "foreground",
+    2: "window-pixel",
+    3: "desktop",
+  }[escalation.target] as NonNullable<ComputerActResult["escalation"]>["recommended"] | undefined;
+  const reasonCode = {
+    0: "route_unavailable",
+    1: "delivery_failed",
+    2: "effect_unconfirmed",
+    3: "suspected_noop",
+    4: "permission_required",
+  }[escalation.reason];
+  if (!recommended || !reasonCode) {
+    throw new Error("COMPUTER_DRIVER_ERROR: invalid CUA Driver action escalation");
+  }
+  return { recommended, reasonCode };
+}
+
+function driverActionDetails(result: CuaToolResult): Record<string, unknown> | undefined {
+  const action = result.action;
+  if (!action) {
+    return undefined;
+  }
+  const details: Record<string, unknown> = {
+    route: [
+      "accessibility",
+      "synthetic_events",
+      "global_input",
+      "system_api",
+      "dom",
+      "trusted_input",
+    ][action.route],
+  };
+  if (action.effect === PARTIAL_EFFECT) {
+    details.partial = true;
+  }
+  if (action.delivery) {
+    details.deliveryMode = ["background", "foreground", "not_applicable", "unknown"][
+      action.delivery.mode
+    ];
+    if (action.delivery.deliveredCount !== undefined) {
+      details.deliveredCount = action.delivery.deliveredCount;
+    }
+  }
+  if (action.evidence?.length) {
+    details.evidence = action.evidence.map(({ kind }) =>
+      kind === VALUE_READBACK_EVIDENCE ? "value_readback" : "window_change",
+    );
+  }
+  return Object.values(details).some((value) => value !== undefined) ? details : undefined;
+}
+
+export function actionEnvelope(
+  result: CuaToolResult,
+  details?: Record<string, unknown>,
+): ComputerActResult {
+  const effect = driverEffect(result);
+  const escalation = driverEscalation(result);
+  const driverDetails = driverActionDetails(result);
+  return {
+    ok: true,
+    ...(effect ? { effect } : {}),
+    ...(escalation ? { escalation } : {}),
+    ...(driverDetails || details ? { details: { ...driverDetails, ...details } } : {}),
+  };
+}
+
+export async function callWindowTool(
+  driver: CuaDriverSession,
+  state: CuaFrameState,
+  name: string,
+  args: Record<string, unknown>,
+  signal?: AbortSignal,
+): Promise<CuaToolResult> {
+  const result = await driver.callTool(name, args, signal);
+  adoptGeneration(state, driver.generation);
+  if (result.isError) {
+    const code = result.errorCode
+      ? `COMPUTER_REFUSED_${result.errorCode}`
+      : "COMPUTER_DRIVER_ERROR";
+    throw new Error(`${code}: ${result.text || `${name} failed`}`);
+  }
+  return result;
+}
+
+export function projectedToolDetails(result: CuaToolResult, tool: string): Record<string, unknown> {
+  if (!result.structuredJson) {
+    throw new Error(`COMPUTER_DRIVER_ERROR: ${tool} returned no structuredContent`);
+  }
+  try {
+    const value: unknown = JSON.parse(result.structuredJson);
+    if (value && typeof value === "object" && !Array.isArray(value)) {
+      return value as Record<string, unknown>;
+    }
+  } catch {}
+  throw new Error(`COMPUTER_DRIVER_ERROR: ${tool} returned invalid structuredContent`);
+}
+
+export function nativeWindows(value: unknown): Array<z.infer<typeof NativeWindowSchema>> {
+  if (!Array.isArray(value)) {
+    return [];
+  }
+  return value.flatMap((entry) => {
+    const parsed = NativeWindowSchema.safeParse(entry);
+    return parsed.success && parsed.data.pid ? [parsed.data] : [];
+  });
+}
+
+export function projectWindows(
+  state: CuaFrameState,
+  windows: Array<z.infer<typeof NativeWindowSchema>>,
+): { windows: Array<Record<string, unknown>>; truncatedWindows?: number } {
+  const bounded = boundedItems(windows);
+  return {
+    windows: bounded.items.map((window) => ({
+      windowRef: issueWindowRef(state, { pid: window.pid!, windowId: window.window_id }),
+      ...(window.app_name ? { appName: window.app_name } : {}),
+      ...(window.title ? { title: window.title } : {}),
+      bounds: window.bounds,
+      ...(window.is_on_screen !== undefined ? { isOnScreen: window.is_on_screen } : {}),
+      ...(window.minimized !== undefined ? { minimized: window.minimized } : {}),
+      ...(window.z_index !== undefined ? { zIndex: window.z_index } : {}),
+    })),
+    ...(bounded.truncated ? { truncatedWindows: bounded.truncated } : {}),
+  };
+}
+
+export function projectApps(state: CuaFrameState, value: unknown): Record<string, unknown> {
+  const raw = Array.isArray(value) ? value : [];
+  const apps = raw.flatMap((entry) => {
+    const parsed = NativeAppSchema.safeParse(entry);
+    if (!parsed.success) {
+      return [];
+    }
+    const app = issueAppRef(state, {
+      ...(parsed.data.pid ? { pid: parsed.data.pid } : {}),
+      name: parsed.data.name,
+      ...(parsed.data.bundle_id ? { bundleId: parsed.data.bundle_id } : {}),
+      ...(parsed.data.launch_path ? { launchPath: parsed.data.launch_path } : {}),
+    });
+    return [
+      {
+        app,
+        name: parsed.data.name,
+        ...(parsed.data.running !== undefined ? { running: parsed.data.running } : {}),
+        ...(parsed.data.active !== undefined ? { active: parsed.data.active } : {}),
+        ...(parsed.data.kind ? { kind: parsed.data.kind } : {}),
+        ...(parsed.data.last_used ? { lastUsed: parsed.data.last_used } : {}),
+      },
+    ];
+  });
+  const bounded = boundedItems(apps);
+  return {
+    apps: bounded.items,
+    totalApps: apps.length,
+    ...(bounded.truncated ? { truncatedApps: bounded.truncated } : {}),
+  };
+}
+
+export function projectProcesses(value: unknown): Record<string, unknown> {
+  const processes = boundedItems(Array.isArray(value) ? value : []);
+  return {
+    processes: processes.items,
+    ...(processes.truncated ? { truncatedProcesses: processes.truncated } : {}),
+  };
+}
+
+export function windowObservation(
+  result: CuaToolResult,
+  state: CuaFrameState,
+  windowRef: string,
+  options: { fromZoom?: boolean } = {},
+): ComputerActResult {
+  const structured = projectedToolDetails(result, options.fromZoom ? "zoom" : "get_window_state");
+  const observation = issueObservation(state, windowRef, options);
+  const snapshotId =
+    typeof structured.snapshot_id === "string" ? structured.snapshot_id : undefined;
+  const rawElements = Array.isArray(structured.elements) ? structured.elements : [];
+  let omittedElementCount = 0;
+  const elements = rawElements.slice(0, 2_000).flatMap((entry) => {
+    const parsed = NativeElementSchema.safeParse(entry);
+    if (!parsed.success || !parsed.data.frame) {
+      omittedElementCount += 1;
+      return [];
+    }
+    const elementRef = issueElementRef(observation, {
+      elementIndex: parsed.data.element_index,
+      ...(parsed.data.element_token ? { elementToken: parsed.data.element_token } : {}),
+      ...(snapshotId ? { snapshotId } : {}),
+    });
+    return [
+      {
+        elementRef,
+        role: parsed.data.role?.trim() || "unknown",
+        ...(parsed.data.label !== undefined ? { label: parsed.data.label } : {}),
+        ...(parsed.data.value !== undefined ? { value: parsed.data.value } : {}),
+        bounds: {
+          x: parsed.data.frame.x,
+          y: parsed.data.frame.y,
+          width: parsed.data.frame.w,
+          height: parsed.data.frame.h,
+        },
+      },
+    ];
+  });
+  const image = result.images.find((entry) => entry.mimeType === "image/png");
+  const base64 = image ? canonicalizeBase64(image.dataBase64) : undefined;
+  if (image && !base64) {
+    throw new Error("COMPUTER_DRIVER_ERROR: CUA Driver returned malformed window PNG base64");
+  }
+  const width =
+    typeof structured.screenshot_width === "number" && structured.screenshot_width > 0
+      ? Math.trunc(structured.screenshot_width)
+      : undefined;
+  const height =
+    typeof structured.screenshot_height === "number" && structured.screenshot_height > 0
+      ? Math.trunc(structured.screenshot_height)
+      : undefined;
+  const details: Record<string, unknown> = {
+    ...(typeof structured.total_element_count === "number"
+      ? { totalElementCount: structured.total_element_count }
+      : {}),
+    ...(rawElements.length > 2_000 ? { truncatedElements: rawElements.length - 2_000 } : {}),
+    ...(omittedElementCount ? { omittedElementsWithoutBounds: omittedElementCount } : {}),
+    ...(structured.degraded === true ? { degraded: true } : {}),
+    ...(typeof structured.degraded_reason === "string"
+      ? { degradedReason: structured.degraded_reason }
+      : {}),
+    ...(typeof structured.screenshot_error === "string"
+      ? { screenshotError: structured.screenshot_error }
+      : {}),
+  };
+  const action = actionEnvelope(result, details);
+  return {
+    ...action,
+    observation: {
+      kind: "window",
+      ...(base64 ? { base64, format: "png" as const } : {}),
+      ...(width ? { width } : {}),
+      ...(height ? { height } : {}),
+      observationId: observation.id,
+      ...(elements.length ? { elements } : {}),
+    },
+    ...(!action.escalation && structured.escalation && typeof structured.escalation === "object"
+      ? { escalation: { recommended: "window-pixel", reasonCode: "ax_tree_unavailable" } }
+      : {}),
+  };
+}

extensions/cua-computer/src/frame.ts

@@ -1,4 +1,4 @@
-import { createHash } from "node:crypto";
+import { createHash, randomUUID } from "node:crypto";
 
 export type CuaDesktopGeometry = {
   platform: string;
@@ -28,12 +28,147 @@ export type CuaLastFrame = {
 export type CuaFrameState = {
   generation: string;
   lastFrame?: CuaLastFrame;
+  apps?: Map<string, CuaAppTarget>;
+  windows?: Map<string, CuaWindowTarget>;
+  observation?: CuaObservationState;
+};
+
+type CuaAppTarget = {
+  pid?: number;
+  name: string;
+  bundleId?: string;
+  launchPath?: string;
+};
+
+type CuaWindowTarget = {
+  pid: number;
+  windowId: number;
+};
+
+type CuaElementTarget = {
+  elementIndex: number;
+  elementToken?: string;
+  snapshotId?: string;
+};
+
+type CuaObservationState = {
+  id: string;
+  windowRef: string;
+  fromZoom: boolean;
+  elements: Map<string, CuaElementTarget>;
 };
 
 function staleFrame(message: string): Error {
   return new Error(`COMPUTER_STALE_FRAME: ${message}; take a new screenshot`);
 }
 
+function staleObservation(): Error {
+  return new Error("COMPUTER_STALE_OBSERVATION: take a fresh observation and retry");
+}
+
+function opaqueRef(kind: "app" | "window" | "observation" | "element"): string {
+  return `cua:v2:${kind}:${randomUUID()}`;
+}
+
+export function adoptGeneration(state: CuaFrameState, generation: string): void {
+  // Native session replacement invalidates every authority-bearing reference,
+  // even when the same window ids and display geometry reappear.
+  if (state.generation !== generation) {
+    state.lastFrame = undefined;
+    state.apps = undefined;
+    state.windows = undefined;
+    state.observation = undefined;
+  }
+  state.generation = generation;
+}
+
+export function verifyGeneration(state: CuaFrameState, generation: string): void {
+  if (state.generation !== generation) {
+    adoptGeneration(state, generation);
+    throw staleObservation();
+  }
+}
+
+export function issueAppRef(state: CuaFrameState, target: CuaAppTarget): string {
+  state.apps ??= new Map();
+  const ref = opaqueRef("app");
+  state.apps.set(ref, target);
+  return ref;
+}
+
+export function resolveAppRef(state: CuaFrameState, ref: string): CuaAppTarget | undefined {
+  return state.apps?.get(ref);
+}
+
+export function issueWindowRef(state: CuaFrameState, target: CuaWindowTarget): string {
+  state.windows ??= new Map();
+  for (const [ref, current] of state.windows) {
+    if (current.pid === target.pid && current.windowId === target.windowId) {
+      return ref;
+    }
+  }
+  const ref = opaqueRef("window");
+  state.windows.set(ref, target);
+  return ref;
+}
+
+export function resolveWindowRef(state: CuaFrameState, ref: string): CuaWindowTarget {
+  const target = state.windows?.get(ref);
+  if (!target) {
+    throw staleObservation();
+  }
+  return target;
+}
+
+export function issueObservation(
+  state: CuaFrameState,
+  windowRef: string,
+  options: { fromZoom?: boolean } = {},
+): CuaObservationState {
+  // Only the newest observation may authorize element or window-pixel actions;
+  // retaining older element tokens would bypass the driver's snapshot lifecycle.
+  const observation: CuaObservationState = {
+    id: opaqueRef("observation"),
+    windowRef,
+    fromZoom: options.fromZoom === true,
+    elements: new Map(),
+  };
+  state.observation = observation;
+  return observation;
+}
+
+export function issueElementRef(
+  observation: CuaObservationState,
+  target: CuaElementTarget,
+): string {
+  const ref = opaqueRef("element");
+  observation.elements.set(ref, target);
+  return ref;
+}
+
+export function resolveObservation(
+  state: CuaFrameState,
+  observationId: string,
+  windowRef: string,
+): CuaObservationState {
+  const observation = state.observation;
+  if (!observation || observation.id !== observationId || observation.windowRef !== windowRef) {
+    throw staleObservation();
+  }
+  return observation;
+}
+
+export function resolveElementRef(
+  observation: CuaObservationState,
+  elementRef: string,
+): CuaElementTarget {
+  const target = observation.elements.get(elementRef);
+  if (!target) {
+    throw staleObservation();
+  }
+  return target;
+}
+
 /**
  * CUA Driver exposes only the primary-display label, not a stable display ID.
  * Bind authorization to connection generation plus the complete live geometry.

extensions/cua-computer/src/v2-actions.ts

@@ -0,0 +1,524 @@
+import {
+  COMPUTER_USE_V2_ACTION_NAMES,
+  type ComputerActParams,
+} from "openclaw/plugin-sdk/computer-use";
+import { normalizeModifiers, parseKeyChord } from "./actions.js";
+import { EscalationReason, type CuaDriverSession } from "./driver-client.js";
+import {
+  actionEnvelope,
+  callWindowTool,
+  nativeWindows,
+  projectApps,
+  projectedToolDetails,
+  projectProcesses,
+  projectWindows,
+  windowObservation,
+} from "./driver-result.js";
+import {
+  adoptGeneration,
+  resolveAppRef,
+  resolveElementRef,
+  resolveObservation,
+  resolveWindowRef,
+  verifyGeneration,
+  type CuaFrameState,
+} from "./frame.js";
+
+const CUA_WIRE_ACTION_NAMES = COMPUTER_USE_V2_ACTION_NAMES.slice(1, 14);
+const CUA_TARGETED_ACTION_NAMES = new Set([
+  "left_click",
+  "right_click",
+  "middle_click",
+  "double_click",
+  "triple_click",
+  "left_click_drag",
+  "left_mouse_down",
+  "left_mouse_up",
+  "scroll",
+  "type",
+  "key",
+] as const);
+
+export type CuaComputerActParams = {
+  action: ComputerActParams["action"];
+  displayFrameId?: string;
+  x?: number;
+  y?: number;
+  fromX?: number;
+  fromY?: number;
+  text?: string;
+  keys?: string;
+  modifiers?: string;
+  scrollDirection?: "up" | "down" | "left" | "right";
+  scrollAmount?: number;
+  durationMs?: number;
+  screenIndex?: number;
+  refWidth?: number;
+  windowRef?: string;
+  elementRef?: string;
+  observationId?: string;
+  deliveryMode?: "background" | "foreground";
+  query?: string;
+  depth?: number;
+  maxElements?: number;
+  app?: string;
+  value?: string;
+  path?: string[];
+  x1?: number;
+  y1?: number;
+  x2?: number;
+  y2?: number;
+  reason?:
+    | "ax_tree_pixel_mismatch"
+    | "background_delivery_failed"
+    | "foreground_ineffective"
+    | "no_window_target"
+    | "other";
+};
+
+function requireWindowTarget(
+  driver: CuaDriverSession,
+  state: CuaFrameState,
+  params: CuaComputerActParams,
+) {
+  verifyGeneration(state, driver.generation);
+  if (!params.windowRef) {
+    throw new Error(`COMPUTER_INVALID_REQUEST: windowRef is required for ${params.action}`);
+  }
+  return {
+    ref: params.windowRef,
+    target: resolveWindowRef(state, params.windowRef),
+  };
+}
+
+function observationTarget(state: CuaFrameState, params: CuaComputerActParams, windowRef: string) {
+  if (!params.observationId) {
+    throw new Error(`COMPUTER_STALE_OBSERVATION: observationId is required for ${params.action}`);
+  }
+  return resolveObservation(state, params.observationId, windowRef);
+}
+
+function elementArgs(
+  state: CuaFrameState,
+  params: CuaComputerActParams,
+  windowRef: string,
+): Record<string, unknown> | undefined {
+  if (!params.elementRef) {
+    return undefined;
+  }
+  const observation = observationTarget(state, params, windowRef);
+  const element = resolveElementRef(observation, params.elementRef);
+  return element.elementToken
+    ? { element_token: element.elementToken }
+    : {
+        element_index: element.elementIndex,
+        ...(element.snapshotId ? { snapshot_id: element.snapshotId } : {}),
+      };
+}
+
+function windowPointArgs(
+  state: CuaFrameState,
+  params: CuaComputerActParams,
+  windowRef: string,
+  point: { x?: number; y?: number },
+  label: string,
+): Record<string, unknown> {
+  if (point.x === undefined || point.y === undefined) {
+    throw new Error(`COMPUTER_INVALID_REQUEST: ${label} coordinates are required`);
+  }
+  const observation = observationTarget(state, params, windowRef);
+  return {
+    x: point.x,
+    y: point.y,
+    ...(observation.fromZoom ? { from_zoom: true } : {}),
+  };
+}
+
+async function handleTargetedAct(
+  platform: NodeJS.Platform,
+  driver: CuaDriverSession,
+  state: CuaFrameState,
+  params: CuaComputerActParams,
+  signal?: AbortSignal,
+): Promise<string> {
+  const { ref: windowRef, target } = requireWindowTarget(driver, state, params);
+  const base = { pid: target.pid, window_id: target.windowId };
+  const delivery = params.deliveryMode ? { delivery_mode: params.deliveryMode } : {};
+  const element = elementArgs(state, params, windowRef);
+  let tool: string;
+  let args: Record<string, unknown>;
+
+  switch (params.action) {
+    case "left_click":
+    case "right_click":
+    case "middle_click":
+    case "double_click":
+    case "triple_click": {
+      tool = "click";
+      const button =
+        params.action === "right_click"
+          ? "right"
+          : params.action === "middle_click"
+            ? "middle"
+            : "left";
+      const count = params.action === "double_click" ? 2 : params.action === "triple_click" ? 3 : 1;
+      const modifiers = normalizeModifiers(params.modifiers);
+      args = {
+        ...base,
+        ...(element ?? windowPointArgs(state, params, windowRef, params, "click")),
+        button,
+        count,
+        ...(modifiers.length ? { modifier: modifiers } : {}),
+        ...delivery,
+      };
+      break;
+    }
+    case "left_click_drag": {
+      if (element) {
+        throw new Error("COMPUTER_UNSUPPORTED_ACTION: cua-driver drag has no element target");
+      }
+      tool = "drag";
+      const from = windowPointArgs(
+        state,
+        params,
+        windowRef,
+        { x: params.fromX, y: params.fromY },
+        "drag start",
+      );
+      const to = windowPointArgs(state, params, windowRef, params, "drag end");
+      const modifiers = normalizeModifiers(params.modifiers);
+      args = {
+        ...base,
+        from_x: from.x,
+        from_y: from.y,
+        to_x: to.x,
+        to_y: to.y,
+        ...(from.from_zoom || to.from_zoom ? { from_zoom: true } : {}),
+        ...(params.durationMs === undefined
+          ? {}
+          : { duration_ms: Math.min(10_000, params.durationMs) }),
+        ...(modifiers.length ? { modifier: modifiers } : {}),
+        ...delivery,
+      };
+      break;
+    }
+    case "left_mouse_down": {
+      if (platform !== "linux") {
+        throw new Error("COMPUTER_UNSUPPORTED_ACTION: left_mouse_down is Linux-only");
+      }
+      if (element || params.deliveryMode === "foreground") {
+        throw new Error(
+          "COMPUTER_UNSUPPORTED_ACTION: left_mouse_down supports only background window pixels",
+        );
+      }
+      tool = "mouse_button_down";
+      args = {
+        ...base,
+        ...windowPointArgs(state, params, windowRef, params, "mouse down"),
+        button: "left",
+      };
+      break;
+    }
+    case "left_mouse_up": {
+      if (platform !== "linux") {
+        throw new Error("COMPUTER_UNSUPPORTED_ACTION: left_mouse_up is Linux-only");
+      }
+      if (element || params.deliveryMode === "foreground") {
+        throw new Error(
+          "COMPUTER_UNSUPPORTED_ACTION: left_mouse_up supports only background window pixels",
+        );
+      }
+      tool = "mouse_button_up";
+      args = {
+        ...base,
+        ...(params.x !== undefined || params.y !== undefined
+          ? windowPointArgs(state, params, windowRef, params, "mouse up")
+          : {}),
+      };
+      break;
+    }
+    case "scroll": {
+      if (!params.scrollDirection) {
+        throw new Error("COMPUTER_INVALID_REQUEST: scrollDirection is required for scroll");
+      }
+      if (normalizeModifiers(params.modifiers).length) {
+        throw new Error(
+          "COMPUTER_UNSUPPORTED_ACTION: modifier-held scroll is unsupported by cua-driver",
+        );
+      }
+      tool = "scroll";
+      args = {
+        ...base,
+        ...(element ??
+          (params.x !== undefined || params.y !== undefined
+            ? windowPointArgs(state, params, windowRef, params, "scroll")
+            : {})),
+        direction: params.scrollDirection,
+        by: "line",
+        amount: Math.min(50, params.scrollAmount ?? 3),
+        ...delivery,
+      };
+      break;
+    }
+    case "type": {
+      if (!params.text) {
+        throw new Error("COMPUTER_INVALID_REQUEST: text is required for type");
+      }
+      tool = "type_text";
+      args = {
+        ...base,
+        ...(element ??
+          (params.x !== undefined || params.y !== undefined
+            ? windowPointArgs(state, params, windowRef, params, "type")
+            : {})),
+        text: params.text,
+        ...delivery,
+      };
+      break;
+    }
+    case "key": {
+      const chord = parseKeyChord(params.keys);
+      tool = "press_key";
+      args = {
+        ...base,
+        ...(element ??
+          (params.x !== undefined || params.y !== undefined
+            ? windowPointArgs(state, params, windowRef, params, "key")
+            : {})),
+        key: chord.key,
+        modifiers: chord.modifiers,
+        ...delivery,
+      };
+      break;
+    }
+    default:
+      throw new Error(`COMPUTER_UNSUPPORTED_ACTION: ${params.action}`);
+  }
+
+  const result = await callWindowTool(driver, state, tool, args, signal);
+  return JSON.stringify(actionEnvelope(result));
+}
+
+export async function handleV2Act(
+  platform: NodeJS.Platform,
+  driver: CuaDriverSession,
+  state: CuaFrameState,
+  params: ComputerActParams,
+  handleDesktop: (
+    driver: CuaDriverSession,
+    state: CuaFrameState,
+    params: ComputerActParams,
+    signal?: AbortSignal,
+  ) => Promise<string>,
+  signal?: AbortSignal,
+): Promise<string> {
+  const input = params as CuaComputerActParams & Record<string, unknown>;
+  if (
+    CUA_TARGETED_ACTION_NAMES.has(input.action as never) &&
+    (input.windowRef || input.elementRef)
+  ) {
+    return await handleTargetedAct(platform, driver, state, input, signal);
+  }
+  if ((CUA_WIRE_ACTION_NAMES as readonly string[]).includes(input.action)) {
+    return await handleDesktop(driver, state, params, signal);
+  }
+
+  switch (input.action) {
+    case "list_apps": {
+      const result = await callWindowTool(driver, state, "list_apps", {}, signal);
+      state.apps = new Map();
+      const structured = projectedToolDetails(result, "list_apps");
+      return JSON.stringify({ ok: true, details: projectApps(state, structured.apps) });
+    }
+    case "list_windows": {
+      const result = await callWindowTool(driver, state, "list_windows", {}, signal);
+      const structured = projectedToolDetails(result, "list_windows");
+      return JSON.stringify({
+        ok: true,
+        details: projectWindows(state, nativeWindows(structured.windows)),
+      });
+    }
+    case "get_accessibility_tree": {
+      if (input.windowRef || input.query || input.depth !== undefined || input.maxElements) {
+        throw new Error(
+          "COMPUTER_UNSUPPORTED_ACTION: CUA Driver 0.19.3 exposes get_accessibility_tree only as unfiltered desktop discovery; use get_window_state for a window tree",
+        );
+      }
+      const result = await callWindowTool(driver, state, "get_accessibility_tree", {}, signal);
+      const structured = projectedToolDetails(result, "get_accessibility_tree");
+      return JSON.stringify({
+        ok: true,
+        details: {
+          ...projectWindows(state, nativeWindows(structured.windows)),
+          ...projectProcesses(structured.processes),
+        },
+      });
+    }
+    case "get_cursor_position": {
+      const result = await callWindowTool(driver, state, "get_cursor_position", {}, signal);
+      return JSON.stringify({
+        ok: true,
+        details: projectedToolDetails(result, "get_cursor_position"),
+      });
+    }
+    case "get_window_state": {
+      verifyGeneration(state, driver.generation);
+      const window = resolveWindowRef(state, input.windowRef!);
+      const result = await callWindowTool(
+        driver,
+        state,
+        "get_window_state",
+        {
+          pid: window.pid,
+          window_id: window.windowId,
+          include_screenshot: true,
+          max_elements: input.maxElements ?? 2_000,
+          ...(input.depth !== undefined ? { max_depth: Math.max(1, input.depth) } : {}),
+          ...(input.query ? { query: input.query } : {}),
+        },
+        signal,
+      );
+      return JSON.stringify(windowObservation(result, state, input.windowRef!));
+    }
+    case "launch_app": {
+      verifyGeneration(state, driver.generation);
+      const appName = input.app!;
+      const app = resolveAppRef(state, appName);
+      if (appName.startsWith("cua:v2:app:") && !app) {
+        throw new Error("COMPUTER_STALE_OBSERVATION: refresh list_apps and retry");
+      }
+      const result = await callWindowTool(
+        driver,
+        state,
+        "launch_app",
+        app
+          ? app.launchPath
+            ? { launch_path: app.launchPath }
+            : app.bundleId
+              ? { bundle_id: app.bundleId }
+              : { name: app.name }
+          : { name: appName },
+        signal,
+      );
+      const structured = projectedToolDetails(result, "launch_app");
+      return JSON.stringify({
+        ...actionEnvelope(result),
+        details: {
+          app: projectApps(state, [structured]).apps,
+          ...projectWindows(state, nativeWindows(structured.windows)),
+        },
+      });
+    }
+    case "kill_app": {
+      verifyGeneration(state, driver.generation);
+      const appName = input.app!;
+      const app = resolveAppRef(state, appName);
+      if (!app?.pid) {
+        throw new Error(
+          "COMPUTER_INVALID_REQUEST: kill_app requires a running app reference from list_apps",
+        );
+      }
+      const result = await callWindowTool(driver, state, "kill_app", { pid: app.pid }, signal);
+      return JSON.stringify(actionEnvelope(result, { app: appName }));
+    }
+    case "bring_to_front": {
+      const { target } = requireWindowTarget(driver, state, input);
+      const result = await callWindowTool(
+        driver,
+        state,
+        "bring_to_front",
+        {
+          pid: target.pid,
+          window_id: target.windowId,
+        },
+        signal,
+      );
+      return JSON.stringify(actionEnvelope(result));
+    }
+    case "set_value": {
+      const { ref, target } = requireWindowTarget(driver, state, input);
+      if (input.deliveryMode === "foreground") {
+        throw new Error(
+          "COMPUTER_UNSUPPORTED_ACTION: cua-driver set_value is background accessibility delivery",
+        );
+      }
+      const element = elementArgs(state, input, ref);
+      if (!element) {
+        throw new Error("COMPUTER_INVALID_REQUEST: elementRef is required for set_value");
+      }
+      const result = await callWindowTool(
+        driver,
+        state,
+        "set_value",
+        {
+          pid: target.pid,
+          window_id: target.windowId,
+          ...element,
+          value: input.value,
+        },
+        signal,
+      );
+      return JSON.stringify(actionEnvelope(result));
+    }
+    case "invoke_menu": {
+      const { target } = requireWindowTarget(driver, state, input);
+      if (input.deliveryMode === "foreground") {
+        throw new Error(
+          "COMPUTER_UNSUPPORTED_ACTION: cua-driver invoke_menu is background accessibility delivery",
+        );
+      }
+      const result = await callWindowTool(
+        driver,
+        state,
+        "invoke_menu",
+        {
+          pid: target.pid,
+          window_id: target.windowId,
+          path: input.path,
+        },
+        signal,
+      );
+      return JSON.stringify(actionEnvelope(result));
+    }
+    case "zoom": {
+      const { ref, target } = requireWindowTarget(driver, state, input);
+      resolveObservation(state, input.observationId!, ref);
+      const result = await callWindowTool(
+        driver,
+        state,
+        "zoom",
+        {
+          pid: target.pid,
+          window_id: target.windowId,
+          x1: input.x1,
+          y1: input.y1,
+          x2: input.x2,
+          y2: input.y2,
+        },
+        signal,
+      );
+      return JSON.stringify(windowObservation(result, state, ref, { fromZoom: true }));
+    }
+    case "escalate_scope": {
+      const reason = {
+        ax_tree_pixel_mismatch: EscalationReason.AxTreePixelMismatch,
+        background_delivery_failed: EscalationReason.BackgroundDeliveryFailed,
+        foreground_ineffective: EscalationReason.ForegroundIneffective,
+        no_window_target: EscalationReason.NoWindowTarget,
+        other: EscalationReason.Other,
+      }[input.reason!];
+      const result = await driver.escalateScope(reason, signal);
+      adoptGeneration(state, driver.generation);
+      return JSON.stringify({
+        ok: true,
+        details: {
+          captureScope: result.captureScope,
+          effectiveScope: result.effectiveScope,
+          desktopUnlocked: result.desktopUnlocked,
+        },
+      });
+    }
+    default:
+      throw new Error(`COMPUTER_UNSUPPORTED_ACTION: ${input.action}`);
+  }
+}