refactor(computer-use): one canonical wire contract + node-host provider seam (#123509)

* refactor(computer-use): add provider seam

* refactor(computer-use): retry provider open after failure; drop changelog entry

docs/plugins/sdk-entrypoints.md

@@ -185,6 +185,22 @@ export default definePluginEntry({
   startup config; command handlers should still validate availability when
   invoked.
 
+### Computer Use providers
+
+**Import:** `openclaw/plugin-sdk/computer-use`
+
+Node-local Computer Use plugins register one provider through
+`registerComputerUseProvider(api, provider)`. The helper owns the
+`screen.snapshot` and dangerous `computer.act` command registrations and the
+matching Gateway invoke policy; the provider owns availability, execution,
+serialization, frame state, driver lifecycle, and cleanup.
+
+The same entry point exports the canonical TypeBox schemas, static types, and
+compiled validators for the two command payloads and the snapshot result. A
+node host accepts one provider for the command pair; registering another
+provider conflicts with the existing command registration instead of creating
+a fallback stack.
+
 ## `defineChannelPluginEntry`
 
 **Import:** `openclaw/plugin-sdk/channel-core`

docs/plugins/sdk-overview.md

@@ -154,6 +154,11 @@ or fully dynamic tool registration.
 | `api.registerCommand(def)`             | Custom command (bypasses the LLM)                                                                                                        |
 | `api.registerNodeHostCommand(command)` | Command handled by `openclaw node run`; optional `agentTool` metadata can expose it as an agent-visible tool while the node is connected |
 
+Computer Use providers use `registerComputerUseProvider(api, provider)` from
+`openclaw/plugin-sdk/computer-use`. It registers the shared
+`screen.snapshot`/`computer.act` node-host envelope once while the provider
+keeps its driver, frame, availability, and execution lifecycle local.
+
 Plugin commands can set `agentPromptGuidance` when the agent needs a short,
 command-owned routing hint. Keep that text about the command itself; do not add
 provider- or plugin-specific policy to core prompt builders.

extensions/cua-computer/index.ts

@@ -1,6 +1,7 @@
+import { registerComputerUseProvider } from "openclaw/plugin-sdk/computer-use";
 import { buildPluginConfigSchema, definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
 import { z } from "zod";
-import { createCuaComputerCommands } from "./src/commands.js";
+import { createCuaComputerProvider } from "./src/commands.js";
 
 const CuaComputerConfigSchema = z.strictObject({
   // Keep the shipped daemon setting as a named no-op: strict validation accepts
@@ -22,17 +23,6 @@ export default definePluginEntry({
         `Invalid cua-computer plugin config: ${parsed.error.issues[0]?.message ?? "invalid config"}`,
       );
     }
-    for (const command of createCuaComputerCommands()) {
-      api.registerNodeHostCommand(command);
-    }
-    // computer.act is dangerous-by-default and therefore also requires the
-    // operator's explicit gateway.nodes.commands.allow entry. The plugin
-    // policy is the final Gateway guard and the only path that may forward the
-    // already-allowlisted invocation to the paired node.
-    api.registerNodeInvokePolicy({
-      commands: ["computer.act"],
-      dangerous: true,
-      handle: async (ctx) => await ctx.invokeNode(),
-    });
+    registerComputerUseProvider(api, createCuaComputerProvider());
   },
 });

extensions/cua-computer/src/actions.ts

@@ -1,41 +1,5 @@
-import { z } from "zod";
 import type { CuaLastFrame } from "./frame.js";
 
-const COMPUTER_ACTIONS = [
-  "left_click",
-  "right_click",
-  "middle_click",
-  "double_click",
-  "triple_click",
-  "mouse_move",
-  "left_click_drag",
-  "left_mouse_down",
-  "left_mouse_up",
-  "scroll",
-  "type",
-  "key",
-  "hold_key",
-] as const;
-
-export const ComputerActParamsSchema = z.strictObject({
-  action: z.enum(COMPUTER_ACTIONS),
-  displayFrameId: z.string().optional(),
-  x: z.number().finite().nonnegative().optional(),
-  y: z.number().finite().nonnegative().optional(),
-  fromX: z.number().finite().nonnegative().optional(),
-  fromY: z.number().finite().nonnegative().optional(),
-  text: z.string().optional(),
-  keys: z.string().optional(),
-  modifiers: z.string().optional(),
-  scrollDirection: z.enum(["up", "down", "left", "right"]).optional(),
-  scrollAmount: z.number().int().positive().optional(),
-  durationMs: z.number().int().nonnegative().optional(),
-  screenIndex: z.number().int().nonnegative().optional(),
-  refWidth: z.number().int().positive().optional(),
-});
-
-export type ComputerActParams = z.infer<typeof ComputerActParamsSchema>;
-
 const MODIFIER_ALIASES = new Map<string, string>([
   ["ctrl", "ctrl"],
   ["control", "ctrl"],

extensions/cua-computer/src/commands.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it, vi } from "vitest";
-import { createCuaComputerCommands } from "./commands.js";
+import { createCuaComputerProvider } from "./commands.js";
 import {
   ClickButton,
   ScrollDirection,
@@ -68,25 +68,25 @@ function driver() {
   };
 }
 
-function commands(session: CuaDriverSession) {
-  return createCuaComputerCommands({
+async function execution(session: CuaDriverSession) {
+  return await createCuaComputerProvider({
     platform: "linux",
     driver: session,
     imageProcessor: {
       encode: vi.fn(async () => ({ data: Buffer.from("jpeg"), width: 100, height: 50 })),
     },
-  });
+  }).openExecution({});
 }
 
-describe("cua-computer direct SDK commands", () => {
+describe("cua-computer provider", () => {
   it("uses one typed session for snapshot and frame-authorized click", async () => {
     const { session, getDesktopState, getScreenSize, click } = driver();
-    const [snapshot, act] = commands(session);
-    const screen = JSON.parse(await snapshot!.handle('{"format":"png","maxWidth":100}')) as {
+    const computer = await execution(session);
+    const screen = JSON.parse(await computer.snapshot('{"format":"png","maxWidth":100}')) as {
       displayFrameId: string;
       width: number;
     };
-    await act!.handle(
+    await computer.act(
       JSON.stringify({
         action: "left_click",
         displayFrameId: screen.displayFrameId,
@@ -110,9 +110,9 @@ describe("cua-computer direct SDK commands", () => {
 
   it("maps scroll and key through typed SDK enums", async () => {
     const { session, typeText, pressKey } = driver();
-    const [, act] = commands(session);
-    await act!.handle('{"action":"type","text":"hello"}');
-    await act!.handle('{"action":"key","keys":"ctrl+enter"}');
+    const computer = await execution(session);
+    await computer.act('{"action":"type","text":"hello"}');
+    await computer.act('{"action":"key","keys":"ctrl+enter"}');
     expect(typeText).toHaveBeenCalledWith("hello", undefined);
     expect(pressKey).toHaveBeenCalledWith({ key: "enter", modifiers: ["ctrl"] }, undefined);
     expect(ScrollDirection.Down).toBeTypeOf("number");
@@ -120,14 +120,14 @@ describe("cua-computer direct SDK commands", () => {
 
   it("maps all remaining projected desktop actions through direct SDK methods", async () => {
     const { session, scroll, moveCursor, drag } = driver();
-    const [snapshot, act] = commands(session);
-    const screen = JSON.parse(await snapshot!.handle('{"format":"png","maxWidth":100}')) as {
+    const computer = await execution(session);
+    const screen = JSON.parse(await computer.snapshot('{"format":"png","maxWidth":100}')) as {
       displayFrameId: string;
       width: number;
     };
     const frame = { displayFrameId: screen.displayFrameId, refWidth: screen.width };
 
-    await act!.handle(
+    await computer.act(
       JSON.stringify({
         action: "scroll",
         ...frame,
@@ -137,8 +137,8 @@ describe("cua-computer direct SDK commands", () => {
         scrollAmount: 4,
       }),
     );
-    await act!.handle(JSON.stringify({ action: "mouse_move", ...frame, x: 11, y: 21 }));
-    await act!.handle(
+    await computer.act(JSON.stringify({ action: "mouse_move", ...frame, x: 11, y: 21 }));
+    await computer.act(
       JSON.stringify({
         action: "left_click_drag",
         ...frame,
@@ -169,13 +169,13 @@ describe("cua-computer direct SDK commands", () => {
       errorCode: "desktop_unavailable",
       text: "desktop input is unavailable",
     });
-    const [snapshot, act] = commands(session);
-    const screen = JSON.parse(await snapshot!.handle('{"format":"png","maxWidth":100}')) as {
+    const computer = await execution(session);
+    const screen = JSON.parse(await computer.snapshot('{"format":"png","maxWidth":100}')) as {
       displayFrameId: string;
       width: number;
     };
     await expect(
-      act!.handle(
+      computer.act(
         JSON.stringify({
           action: "left_click",
           displayFrameId: screen.displayFrameId,
@@ -189,14 +189,14 @@ describe("cua-computer direct SDK commands", () => {
 
   it("rejects a mismatched reference width before desktop input", async () => {
     const { session, click } = driver();
-    const [snapshot, act] = commands(session);
-    const screen = JSON.parse(await snapshot!.handle('{"format":"png","maxWidth":100}')) as {
+    const computer = await execution(session);
+    const screen = JSON.parse(await computer.snapshot('{"format":"png","maxWidth":100}')) as {
       displayFrameId: string;
       width: number;
     };
 
     await expect(
-      act!.handle(
+      computer.act(
         JSON.stringify({
           action: "left_click",
           displayFrameId: screen.displayFrameId,
@@ -213,7 +213,7 @@ describe("cua-computer direct SDK commands", () => {
     const { session, dispose } = driver();
     const createDriver = vi.fn(() => session);
     const clearInterval = vi.fn();
-    const [snapshot] = createCuaComputerCommands({
+    const provider = createCuaComputerProvider({
       platform: "linux",
       createDriver,
       imageProcessor: {
@@ -224,10 +224,11 @@ describe("cua-computer direct SDK commands", () => {
     });
     expect(createDriver).not.toHaveBeenCalled();
 
-    await snapshot!.handle('{"format":"png","maxWidth":100}');
+    const computer = await provider.openExecution({});
+    await computer.snapshot('{"format":"png","maxWidth":100}');
     expect(createDriver).toHaveBeenCalledOnce();
 
-    const stop = snapshot!.watchAvailability?.({ config: {} as never, env: {} }, vi.fn());
+    const stop = provider.watchAvailability?.({ config: {} as never, env: {} }, vi.fn());
     stop?.();
     await Promise.resolve();
     expect(clearInterval).toHaveBeenCalledOnce();
@@ -236,12 +237,9 @@ describe("cua-computer direct SDK commands", () => {
 
   it("passes node invocation cancellation to the direct SDK", async () => {
     const { session, getDesktopState } = driver();
-    const [snapshot] = commands(session);
+    const computer = await execution(session);
     const signal = AbortSignal.abort();
-    await snapshot!.handle('{"format":"png","maxWidth":100}', undefined, {
-      sendNodeEvent: vi.fn(),
-      signal,
-    });
+    await computer.snapshot('{"format":"png","maxWidth":100}', signal);
     expect(getDesktopState).toHaveBeenCalledWith(signal);
   });
 });

extensions/cua-computer/src/commands.ts

@@ -1,17 +1,16 @@
 import fs from "node:fs";
 import path from "node:path";
+import {
+  parseComputerActParamsJSON,
+  parseScreenSnapshotParamsJSON,
+  type ComputerActParams,
+  type ComputerUseProvider,
+} from "openclaw/plugin-sdk/computer-use";
 import { canonicalizeBase64 } from "openclaw/plugin-sdk/media-runtime";
-import type { OpenClawPluginNodeHostCommand } from "openclaw/plugin-sdk/plugin-entry";
 import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path";
 import { createRastermill } from "rastermill";
 import { z } from "zod";
-import {
-  ComputerActParamsSchema,
-  normalizeModifiers,
-  parseKeyChord,
-  scalePoint,
-  type ComputerActParams,
-} from "./actions.js";
+import { normalizeModifiers, parseKeyChord, scalePoint } from "./actions.js";
 import {
   ClickButton,
   ScrollDirection,
@@ -35,13 +34,6 @@ const AVAILABILITY_POLL_MS = 5_000;
 // display; budget above it so full-resolution snapshots reach the downscaler.
 const MAX_IMAGE_PIXELS = 40_000_000;
 
-const SnapshotParamsSchema = z.strictObject({
-  screenIndex: z.number().int().nonnegative().optional(),
-  maxWidth: z.number().int().positive().optional(),
-  quality: z.number().finite().optional(),
-  format: z.enum(["jpeg", "png"]).optional(),
-});
-
 const DesktopStateSchema = z.object({
   platform: z.string().min(1),
   display: z.string().min(1),
@@ -69,7 +61,7 @@ type ImageProcessor = {
   ): Promise<{ data: Buffer; width: number; height: number }>;
 };
 
-type CuaComputerCommandsOptions = {
+type CuaComputerProviderOptions = {
   platform?: NodeJS.Platform;
   env?: NodeJS.ProcessEnv;
   driver?: CuaDriverSession;
@@ -97,22 +89,6 @@ class PromiseQueue {
   }
 }
 
-function parseParams<T>(schema: z.ZodType<T>, paramsJSON: string | null | undefined): T {
-  let value: unknown;
-  try {
-    value = JSON.parse(paramsJSON ?? "{}");
-  } catch {
-    throw new Error("COMPUTER_INVALID_REQUEST: params must be valid JSON");
-  }
-  const parsed = schema.safeParse(value);
-  if (!parsed.success) {
-    throw new Error(
-      `COMPUTER_INVALID_REQUEST: ${parsed.error.issues[0]?.message ?? "invalid params"}`,
-    );
-  }
-  return parsed.data;
-}
-
 function assertPrimaryDisplay(screenIndex: number | undefined): void {
   if (screenIndex !== undefined && screenIndex !== 0) {
     throw new Error(
@@ -407,9 +383,9 @@ async function handleAct(
   return JSON.stringify({ ok: true });
 }
 
-export function createCuaComputerCommands(
-  options: CuaComputerCommandsOptions = {},
-): OpenClawPluginNodeHostCommand[] {
+export function createCuaComputerProvider(
+  options: CuaComputerProviderOptions = {},
+): ComputerUseProvider {
   const platform = options.platform ?? process.platform;
   const env = options.env ?? process.env;
   let ownedDriver: CuaDriverSession | undefined;
@@ -429,17 +405,14 @@ export function createCuaComputerCommands(
     await current?.dispose();
   };
   const imageProcessor = options.imageProcessor ?? createImageProcessor(env);
-  const queue = new PromiseQueue();
-  const frameState: CuaFrameState = { generation: "uninitialized" };
   const interval = options.setInterval ?? setInterval;
   const clear = options.clearInterval ?? clearInterval;
   const isSupportedPlatform = platform === "linux" || platform === "win32";
   const isAvailable = () => isSupportedPlatform && driver().isAvailable();
 
-  const snapshot: OpenClawPluginNodeHostCommand = {
-    command: "screen.snapshot",
-    cap: "screen",
-    dangerous: false,
+  return {
+    id: "cua-computer",
+    label: "CUA Computer",
     isAvailable,
     watchAvailability: (_context, onChange) => {
       let knownAvailable = isAvailable();
@@ -457,76 +430,78 @@ export function createCuaComputerCommands(
         void disposeOwnedDriver();
       };
     },
-    handle: async (paramsJSON, _io, context) =>
-      await queue.run(async () => {
-        if (!isSupportedPlatform) {
-          throw new Error("COMPUTER_DRIVER_UNAVAILABLE: cua-computer supports Windows and Linux");
-        }
-        const params = parseParams(SnapshotParamsSchema, paramsJSON);
-        assertPrimaryDisplay(params.screenIndex);
-        const format = params.format ?? "jpeg";
-        const maxWidth = params.maxWidth ?? (format === "png" ? 900 : 1_600);
-        const quality = Math.min(1, Math.max(0.05, params.quality ?? 0.72));
-        const desktop = await driver().getDesktopState(context?.signal);
-        const geometry = desktopGeometry(desktop);
-        // cua-driver desktop input consumes native get_desktop_state PNG pixels,
-        // and on every supported backend the driver reports screen geometry in
-        // that same physical-pixel space (Windows PMv2, Linux X11/Wayland). If a
-        // capture ever diverges from screen geometry, our screenshot->native
-        // scaling would mis-target input, so refuse rather than click blind.
-        if (
-          geometry.screenWidth !== geometry.screenshotWidth ||
-          geometry.screenHeight !== geometry.screenshotHeight
-        ) {
-          throw new Error(
-            "COMPUTER_UNSUPPORTED_DISPLAY: cua-driver reported capture and screen geometry in different pixel spaces",
-          );
-        }
-        const nativePng = desktopPng(desktop);
-        let encoded = nativePng;
-        let width = geometry.screenshotWidth;
-        let height = geometry.screenshotHeight;
-        if (format === "jpeg" || width > maxWidth) {
-          const result = await imageProcessor.encode(nativePng, {
-            format,
-            ...(format === "jpeg" ? { quality: Math.round(quality * 100) } : {}),
-            ...(width > maxWidth ? { resize: { width: maxWidth, enlarge: false } } : {}),
-          });
-          encoded = result.data;
-          width = result.width;
-          height = result.height;
-        }
-        frameState.generation = driver().generation;
-        const displayFrameId = issueFrame(frameState, geometry, { width, height });
-        return JSON.stringify({
-          format,
-          base64: encoded.toString("base64"),
-          displayFrameId,
-          screenIndex: 0,
-          width,
-          height,
-        });
-      }),
+    openExecution: async () => {
+      const queue = new PromiseQueue();
+      const frameState: CuaFrameState = { generation: "uninitialized" };
+      return {
+        snapshot: async (paramsJSON, signal) =>
+          await queue.run(async () => {
+            if (!isSupportedPlatform) {
+              throw new Error(
+                "COMPUTER_DRIVER_UNAVAILABLE: cua-computer supports Windows and Linux",
+              );
+            }
+            const params = parseScreenSnapshotParamsJSON(paramsJSON);
+            assertPrimaryDisplay(params.screenIndex);
+            const format = params.format ?? "jpeg";
+            const maxWidth = params.maxWidth ?? (format === "png" ? 900 : 1_600);
+            const quality = Math.min(1, Math.max(0.05, params.quality ?? 0.72));
+            const desktop = await driver().getDesktopState(signal);
+            const geometry = desktopGeometry(desktop);
+            // cua-driver desktop input consumes native get_desktop_state PNG pixels,
+            // and on every supported backend the driver reports screen geometry in
+            // that same physical-pixel space (Windows PMv2, Linux X11/Wayland). If a
+            // capture ever diverges from screen geometry, our screenshot->native
+            // scaling would mis-target input, so refuse rather than click blind.
+            if (
+              geometry.screenWidth !== geometry.screenshotWidth ||
+              geometry.screenHeight !== geometry.screenshotHeight
+            ) {
+              throw new Error(
+                "COMPUTER_UNSUPPORTED_DISPLAY: cua-driver reported capture and screen geometry in different pixel spaces",
+              );
+            }
+            const nativePng = desktopPng(desktop);
+            let encoded = nativePng;
+            let width = geometry.screenshotWidth;
+            let height = geometry.screenshotHeight;
+            if (format === "jpeg" || width > maxWidth) {
+              const result = await imageProcessor.encode(nativePng, {
+                format,
+                ...(format === "jpeg" ? { quality: Math.round(quality * 100) } : {}),
+                ...(width > maxWidth ? { resize: { width: maxWidth, enlarge: false } } : {}),
+              });
+              encoded = result.data;
+              width = result.width;
+              height = result.height;
+            }
+            frameState.generation = driver().generation;
+            const displayFrameId = issueFrame(frameState, geometry, { width, height });
+            return JSON.stringify({
+              format,
+              base64: encoded.toString("base64"),
+              displayFrameId,
+              screenIndex: 0,
+              width,
+              height,
+            });
+          }),
+        act: async (paramsJSON, signal) =>
+          await queue.run(async () => {
+            if (!isSupportedPlatform) {
+              throw new Error(
+                "COMPUTER_DRIVER_UNAVAILABLE: cua-computer supports Windows and Linux",
+              );
+            }
+            return await handleAct(
+              driver(),
+              frameState,
+              parseComputerActParamsJSON(paramsJSON),
+              signal,
+            );
+          }),
+        close: async () => await disposeOwnedDriver(),
+      };
+    },
   };
-
-  const act: OpenClawPluginNodeHostCommand = {
-    command: "computer.act",
-    cap: "computer",
-    dangerous: true,
-    isAvailable,
-    handle: async (paramsJSON, _io, context) =>
-      await queue.run(async () => {
-        if (!isSupportedPlatform) {
-          throw new Error("COMPUTER_DRIVER_UNAVAILABLE: cua-computer supports Windows and Linux");
-        }
-        return await handleAct(
-          driver(),
-          frameState,
-          parseParams(ComputerActParamsSchema, paramsJSON),
-          context?.signal,
-        );
-      }),
-  };
-
-  return [snapshot, act];
 }

package.json

@@ -1065,6 +1065,10 @@
     "./plugin-sdk/node-host": {
       "default": "./dist/plugin-sdk/node-host.js"
     },
+    "./plugin-sdk/computer-use": {
+      "types": "./dist/plugin-sdk/computer-use.d.ts",
+      "default": "./dist/plugin-sdk/computer-use.js"
+    },
     "./plugin-sdk/response-limit-runtime": {
       "default": "./dist/plugin-sdk/response-limit-runtime.js"
     },

scripts/lib/plugin-sdk-entrypoints.json

@@ -209,6 +209,7 @@
   "runtime-fetch",
   "inline-image-data-url-runtime",
   "node-host",
+  "computer-use",
   "response-limit-runtime",
   "session-binding-runtime",
   "session-catalog",

scripts/plugin-sdk-surface-report.mts

@@ -189,7 +189,8 @@ export function readPluginSdkSurfaceBudgets(env: NodeJS.ProcessEnv = process.env
       // +1: dependency-light channel streaming config readers for doctor closures
       //     (realtime-voice-activation is private-local and not counted here).
       // +1: registry-bound plugin command planning and exact selected execution.
-      144,
+      // +1: canonical Computer Use wire contract and node-host provider seam.
+      145,
       env,
     ),
     publicExports: readPluginSdkSurfaceBudgetEnv(
@@ -272,7 +273,8 @@ export function readPluginSdkSurfaceBudgets(env: NodeJS.ProcessEnv = process.env
       // +1: normalized Gateway public origin resolver for plugin-generated links.
       // -2: retire the dead progress-draft render reader; it counted twice via
       // channel-outbound and channel-message's wildcard re-export of it.
-      4306,
+      // +11: Computer Use schemas/types plus parsers, compiler, and provider registration.
+      4317,
       env,
     ),
     publicFunctionExports: readPluginSdkSurfaceBudgetEnv(
@@ -341,7 +343,8 @@ export function readPluginSdkSurfaceBudgets(env: NodeJS.ProcessEnv = process.env
       // +1: normalized Gateway public origin resolver for plugin-generated links.
       // -2: retire the dead progress-draft render reader; it counted twice via
       // channel-outbound and channel-message's wildcard re-export of it.
-      2570,
+      // +4: Computer Use wire parsers, validator compiler, and provider registration.
+      2574,
       env,
     ),
     publicDeprecatedExports: readPluginSdkSurfaceBudgetEnv(

src/agents/tools/computer-tool.ts

@@ -14,6 +14,10 @@ import { Type } from "typebox";
 import { parseScreenSnapshotPayload } from "../../cli/nodes-screen.js";
 import type { OpenClawConfig } from "../../config/types.openclaw.js";
 import { formatErrorMessage } from "../../infra/errors.js";
+import type {
+  ComputerActParams,
+  ScreenSnapshotParams,
+} from "../../plugins/computer-use-contract.js";
 import { sleep } from "../../utils/sleep.js";
 import {
   DEFAULT_IMAGE_MAX_DIMENSION_PX,
@@ -77,7 +81,7 @@ const COMPUTER_TOOL_ACTIONS = [
 
 type ComputerToolAction = (typeof COMPUTER_TOOL_ACTIONS)[number];
 
-const INPUT_ACTIONS = new Set<ComputerToolAction>([
+const INPUT_ACTIONS = new Set<ComputerActParams["action"]>([
   "left_click",
   "right_click",
   "middle_click",
@@ -93,6 +97,10 @@ const INPUT_ACTIONS = new Set<ComputerToolAction>([
   "hold_key",
 ]);
 
+function isComputerActAction(action: ComputerToolAction): action is ComputerActParams["action"] {
+  return INPUT_ACTIONS.has(action as ComputerActParams["action"]);
+}
+
 const COORDINATE_REQUIRED_ACTIONS = new Set<ComputerToolAction>([
   "left_click",
   "right_click",
@@ -126,6 +134,12 @@ const MODIFIER_TEXT_ACTIONS = new Set<ComputerToolAction>([
 
 const SCROLL_DIRECTIONS = ["up", "down", "left", "right"] as const;
 
+function isScrollDirection(
+  value: string,
+): value is NonNullable<ComputerActParams["scrollDirection"]> {
+  return SCROLL_DIRECTIONS.some((direction) => direction === value);
+}
+
 const ComputerToolSchema = Type.Object({
   action: stringEnum(COMPUTER_TOOL_ACTIONS),
   ...gatewayCallOptionSchemaProperties(),
@@ -177,23 +191,6 @@ const ComputerToolSchema = Type.Object({
   ),
 });
 
-type ComputerActWireParams = {
-  action: string;
-  displayFrameId?: string;
-  x?: number;
-  y?: number;
-  fromX?: number;
-  fromY?: number;
-  text?: string;
-  keys?: string;
-  modifiers?: string;
-  scrollDirection?: string;
-  scrollAmount?: number;
-  durationMs?: number;
-  screenIndex?: number;
-  refWidth: number;
-};
-
 function readCoordinate(
   params: Record<string, unknown>,
   key: "coordinate" | "startCoordinate",
@@ -236,14 +233,14 @@ function readModifiers(params: Record<string, unknown>, action: ComputerToolActi
 
 /** Builds the computer.act wire params for one tool input action. */
 function buildComputerActParams(params: {
-  action: ComputerToolAction;
+  action: ComputerActParams["action"];
   input: Record<string, unknown>;
   screenIndex: number;
   displayFrameId?: string;
   refWidth?: number;
-}): ComputerActWireParams {
+}): ComputerActParams {
   const { action, input } = params;
-  const wire: ComputerActWireParams = {
+  const wire: ComputerActParams = {
     action,
     screenIndex: params.screenIndex,
     refWidth: params.refWidth ?? COMPUTER_REF_WIDTH,
@@ -278,7 +275,7 @@ function buildComputerActParams(params: {
     }
     case "scroll": {
       const direction = normalizeOptionalLowercaseString(input.scrollDirection);
-      if (!direction || !SCROLL_DIRECTIONS.includes(direction as never)) {
+      if (!direction || !isScrollDirection(direction)) {
         throw new Error("scrollDirection up|down|left|right required for scroll");
       }
       wire.scrollDirection = direction;
@@ -409,16 +406,17 @@ async function captureScreenshot(params: {
   refWidth: number;
   signal?: AbortSignal;
 }): Promise<ScreenshotCapture> {
+  const commandParams: ScreenSnapshotParams = {
+    screenIndex: params.screenIndex,
+    maxWidth: params.refWidth,
+    quality: SCREENSHOT_QUALITY,
+    format: "jpeg",
+  };
   const payload = await invokeNodeCommand({
     gatewayOpts: params.gatewayOpts,
     nodeId: params.nodeId,
     command: SCREEN_SNAPSHOT_COMMAND,
-    commandParams: {
-      screenIndex: params.screenIndex,
-      maxWidth: params.refWidth,
-      quality: SCREENSHOT_QUALITY,
-      format: "jpeg",
-    },
+    commandParams,
     signal: params.signal,
   });
   const parsed = parseScreenSnapshotPayload(payload);
@@ -861,8 +859,8 @@ export function createComputerTool(options?: {
             break;
         }
 
-        if (!INPUT_ACTIONS.has(action)) {
-          throw new Error(`Unknown action: ${action}`);
+        if (!isComputerActAction(action)) {
+          throw new Error(`Unknown action: ${String(action)}`);
         }
         const wireParams = buildComputerActParams({
           action,

src/cli/nodes-screen.ts

@@ -1,6 +1,10 @@
 // Screen-recording payload helpers for node media commands.
 import * as path from "node:path";
 import { extnameFromAnyPath } from "@openclaw/media-core/file-name";
+import {
+  parseScreenSnapshotResult,
+  type ScreenSnapshotResult,
+} from "../plugins/computer-use-contract.js";
 import { writeBase64ToFile } from "./nodes-camera.js";
 import { asRecord, readStringValue, resolveTempPathParts } from "./nodes-media-utils.js";
 
@@ -48,32 +52,9 @@ export async function writeScreenRecordToFile(
 }
 
 /** Validated payload returned by `nodes screen snapshot` RPC calls. */
-type ScreenSnapshotPayload = {
-  format: string;
-  base64: string;
-  /** Node-issued token binding this image to one physical display geometry. */
-  displayFrameId?: string;
-  screenIndex?: number;
-  width?: number;
-  height?: number;
-};
-
 /** Validate and normalize an unknown screen-snapshot payload. */
-export function parseScreenSnapshotPayload(value: unknown): ScreenSnapshotPayload {
-  const obj = asRecord(value);
-  const format = readStringValue(obj.format);
-  const base64 = readStringValue(obj.base64);
-  if (!format || !base64) {
-    throw new Error("invalid screen.snapshot payload");
-  }
-  return {
-    format,
-    base64,
-    displayFrameId: readStringValue(obj.displayFrameId) || undefined,
-    screenIndex: typeof obj.screenIndex === "number" ? obj.screenIndex : undefined,
-    width: typeof obj.width === "number" ? obj.width : undefined,
-    height: typeof obj.height === "number" ? obj.height : undefined,
-  };
+export function parseScreenSnapshotPayload(value: unknown): ScreenSnapshotResult {
+  return parseScreenSnapshotResult(value);
 }
 
 /**

src/plugin-sdk/computer-use.ts

@@ -0,0 +1,15 @@
+export {
+  ComputerActParamsSchema,
+  ScreenSnapshotParamsSchema,
+  ScreenSnapshotResultSchema,
+  compileComputerUseValidator,
+  parseComputerActParamsJSON,
+  parseScreenSnapshotParamsJSON,
+  registerComputerUseProvider,
+} from "../plugins/computer-use-contract.js";
+export type {
+  ComputerActParams,
+  ComputerUseProvider,
+  ScreenSnapshotParams,
+  ScreenSnapshotResult,
+} from "../plugins/computer-use-contract.js";

src/plugins/computer-use-contract.test.ts

@@ -0,0 +1,102 @@
+import { describe, expect, it, vi } from "vitest";
+import {
+  parseComputerActParamsJSON,
+  parseScreenSnapshotResult,
+  registerComputerUseProvider,
+  type ComputerUseProvider,
+} from "./computer-use-contract.js";
+import type { OpenClawPluginNodeHostCommand, OpenClawPluginNodeInvokePolicy } from "./types.js";
+
+describe("Computer Use wire contract", () => {
+  it("validates the canonical computer.act payload", () => {
+    expect(
+      parseComputerActParamsJSON(
+        JSON.stringify({
+          action: "left_click",
+          displayFrameId: "frame-1",
+          x: 10,
+          y: 20,
+          refWidth: 1280,
+        }),
+      ),
+    ).toEqual({
+      action: "left_click",
+      displayFrameId: "frame-1",
+      x: 10,
+      y: 20,
+      refWidth: 1280,
+    });
+    expect(() => parseComputerActParamsJSON('{"action":"left_click","unexpected":true}')).toThrow(
+      "COMPUTER_INVALID_REQUEST",
+    );
+  });
+
+  it("projects the canonical screen.snapshot result", () => {
+    expect(
+      parseScreenSnapshotResult({
+        format: "jpeg",
+        base64: "aGk=",
+        displayFrameId: "frame-1",
+        width: 100,
+        height: 50,
+        capturedAtMs: 42,
+        ignored: true,
+      }),
+    ).toEqual({
+      format: "jpeg",
+      base64: "aGk=",
+      displayFrameId: "frame-1",
+      width: 100,
+      height: 50,
+      capturedAtMs: 42,
+    });
+  });
+});
+
+describe("Computer Use provider registration", () => {
+  it("registers one command pair and dispatches both through one execution", async () => {
+    const commands: OpenClawPluginNodeHostCommand[] = [];
+    const policies: OpenClawPluginNodeInvokePolicy[] = [];
+    const snapshot = vi.fn(async () => "snapshot");
+    const act = vi.fn(async () => "act");
+    const close = vi.fn(async () => {});
+    const stopWatching = vi.fn();
+    const openExecution = vi.fn(async () => ({ snapshot, act, close }));
+    const provider: ComputerUseProvider = {
+      id: "fixture",
+      label: "Fixture",
+      isAvailable: () => true,
+      watchAvailability: () => stopWatching,
+      openExecution,
+    };
+
+    registerComputerUseProvider(
+      {
+        registerNodeHostCommand: (command) => commands.push(command),
+        registerNodeInvokePolicy: (policy) => policies.push(policy),
+      },
+      provider,
+    );
+
+    expect(commands.map(({ command, cap, dangerous }) => ({ command, cap, dangerous }))).toEqual([
+      { command: "screen.snapshot", cap: "screen", dangerous: false },
+      { command: "computer.act", cap: "computer", dangerous: true },
+    ]);
+    expect(policies).toHaveLength(1);
+    expect(policies[0]).toMatchObject({ commands: ["computer.act"], dangerous: true });
+
+    const signal = new AbortController().signal;
+    const context = { sendNodeEvent: vi.fn(), sessionKey: "session-1", signal };
+    await expect(commands[0]!.handle("{}", undefined, context)).resolves.toBe("snapshot");
+    await expect(commands[1]!.handle("{}", undefined, context)).resolves.toBe("act");
+    expect(openExecution).toHaveBeenCalledOnce();
+    expect(openExecution).toHaveBeenCalledWith({ sessionKey: "session-1" });
+    expect(snapshot).toHaveBeenCalledWith("{}", signal);
+    expect(act).toHaveBeenCalledWith("{}", signal);
+
+    const stop = commands[0]!.watchAvailability?.({ config: {} as never, env: {} }, vi.fn());
+    stop?.();
+    await vi.waitFor(() => expect(close).toHaveBeenCalledWith("node-host-stop"));
+    expect(stopWatching).toHaveBeenCalledOnce();
+  });
+});

src/plugins/computer-use-contract.ts

@@ -0,0 +1,215 @@
+import { type Static, type TSchema, Type } from "typebox";
+import { Compile } from "typebox/compile";
+import type { OpenClawPluginApi } from "./plugin-api.types.js";
+import type {
+  OpenClawPluginNodeHostCommandAvailabilityContext,
+  OpenClawPluginNodeHostCommandContext,
+} from "./types.node-host.js";
+
+const COMPUTER_ACT_ACTIONS = [
+  "left_click",
+  "right_click",
+  "middle_click",
+  "double_click",
+  "triple_click",
+  "mouse_move",
+  "left_click_drag",
+  "left_mouse_down",
+  "left_mouse_up",
+  "scroll",
+  "type",
+  "key",
+  "hold_key",
+] as const;
+
+const SCROLL_DIRECTIONS = ["up", "down", "left", "right"] as const;
+
+/** Canonical inner payload accepted by the `computer.act` node command. */
+export const ComputerActParamsSchema = Type.Object(
+  {
+    action: Type.Enum(COMPUTER_ACT_ACTIONS, { type: "string" }),
+    displayFrameId: Type.Optional(Type.String()),
+    x: Type.Optional(Type.Number({ minimum: 0 })),
+    y: Type.Optional(Type.Number({ minimum: 0 })),
+    fromX: Type.Optional(Type.Number({ minimum: 0 })),
+    fromY: Type.Optional(Type.Number({ minimum: 0 })),
+    text: Type.Optional(Type.String()),
+    keys: Type.Optional(Type.String()),
+    modifiers: Type.Optional(Type.String()),
+    scrollDirection: Type.Optional(Type.Enum(SCROLL_DIRECTIONS, { type: "string" })),
+    scrollAmount: Type.Optional(Type.Integer({ minimum: 1 })),
+    durationMs: Type.Optional(Type.Integer({ minimum: 0 })),
+    screenIndex: Type.Optional(Type.Integer({ minimum: 0 })),
+    refWidth: Type.Optional(Type.Integer({ minimum: 1 })),
+  },
+  { additionalProperties: false },
+);
+
+/** Canonical inner payload accepted by the `screen.snapshot` node command. */
+export const ScreenSnapshotParamsSchema = Type.Object(
+  {
+    screenIndex: Type.Optional(Type.Integer({ minimum: 0 })),
+    maxWidth: Type.Optional(Type.Integer({ minimum: 1 })),
+    quality: Type.Optional(Type.Number()),
+    format: Type.Optional(Type.Enum(["jpeg", "png"], { type: "string" })),
+  },
+  { additionalProperties: false },
+);
+
+/** Canonical inner payload returned by the `screen.snapshot` node command. */
+export const ScreenSnapshotResultSchema = Type.Object({
+  format: Type.Enum(["jpeg", "png"], { type: "string" }),
+  base64: Type.String({ minLength: 1 }),
+  displayFrameId: Type.Optional(Type.String()),
+  screenIndex: Type.Optional(Type.Number()),
+  width: Type.Optional(Type.Number()),
+  height: Type.Optional(Type.Number()),
+  capturedAtMs: Type.Optional(Type.Integer({ minimum: 0 })),
+});
+
+export type ComputerActParams = Static<typeof ComputerActParamsSchema>;
+export type ScreenSnapshotParams = Static<typeof ScreenSnapshotParamsSchema>;
+export type ScreenSnapshotResult = Static<typeof ScreenSnapshotResultSchema>;
+
+type ComputerUseValidator<Value> = (value: unknown) => value is Value;
+
+/** Compile one Computer Use wire schema into a reusable type-guard validator. */
+export function compileComputerUseValidator<const Schema extends TSchema>(
+  schema: Schema,
+): ComputerUseValidator<Static<Schema>> {
+  const validator = Compile(schema);
+  return (value: unknown): value is Static<Schema> => validator.Check(value);
+}
+
+const validateComputerActParams = compileComputerUseValidator(ComputerActParamsSchema);
+const validateScreenSnapshotParams = compileComputerUseValidator(ScreenSnapshotParamsSchema);
+const validateScreenSnapshotResult = compileComputerUseValidator(ScreenSnapshotResultSchema);
+
+function parseParamsJSON<Value>(
+  paramsJSON: string | null | undefined,
+  validate: ComputerUseValidator<Value>,
+): Value {
+  let value: unknown;
+  try {
+    value = JSON.parse(paramsJSON ?? "{}");
+  } catch {
+    throw new Error("COMPUTER_INVALID_REQUEST: params must be valid JSON");
+  }
+  if (!validate(value)) {
+    throw new Error("COMPUTER_INVALID_REQUEST: invalid params");
+  }
+  return value;
+}
+
+export function parseComputerActParamsJSON(
+  paramsJSON: string | null | undefined,
+): ComputerActParams {
+  return parseParamsJSON(paramsJSON, validateComputerActParams);
+}
+
+export function parseScreenSnapshotParamsJSON(
+  paramsJSON: string | null | undefined,
+): ScreenSnapshotParams {
+  return parseParamsJSON(paramsJSON, validateScreenSnapshotParams);
+}
+
+/** Validate and project a `screen.snapshot` result without retaining unknown fields. */
+export function parseScreenSnapshotResult(value: unknown): ScreenSnapshotResult {
+  if (!validateScreenSnapshotResult(value)) {
+    throw new Error("invalid screen.snapshot payload");
+  }
+  return {
+    format: value.format,
+    base64: value.base64,
+    ...(value.displayFrameId ? { displayFrameId: value.displayFrameId } : {}),
+    ...(value.screenIndex !== undefined ? { screenIndex: value.screenIndex } : {}),
+    ...(value.width !== undefined ? { width: value.width } : {}),
+    ...(value.height !== undefined ? { height: value.height } : {}),
+    ...(value.capturedAtMs !== undefined ? { capturedAtMs: value.capturedAtMs } : {}),
+  };
+}
+
+type ComputerUseExecution = {
+  snapshot(paramsJSON: string | null | undefined, signal?: AbortSignal): Promise<string>;
+  act(paramsJSON: string | null | undefined, signal?: AbortSignal): Promise<string>;
+  close(reason: string): Promise<void>;
+};
+
+export type ComputerUseProvider = {
+  id: string;
+  label: string;
+  isAvailable(): boolean;
+  watchAvailability?: (
+    context: OpenClawPluginNodeHostCommandAvailabilityContext,
+    onChange: () => void,
+  ) => (() => void) | void;
+  openExecution(context: { sessionKey?: string }): Promise<ComputerUseExecution>;
+};
+
+type ComputerUseRegistrationApi = Pick<
+  OpenClawPluginApi,
+  "registerNodeHostCommand" | "registerNodeInvokePolicy"
+>;
+
+/** Register the canonical node-host command pair for one node-local provider. */
+export function registerComputerUseProvider(
+  api: ComputerUseRegistrationApi,
+  provider: ComputerUseProvider,
+): void {
+  let executionPromise: Promise<ComputerUseExecution> | undefined;
+
+  const getExecution = (context?: OpenClawPluginNodeHostCommandContext) => {
+    if (!executionPromise) {
+      const opened = provider.openExecution(
+        context?.sessionKey ? { sessionKey: context.sessionKey } : {},
+      );
+      // A failed open must not wedge the provider behind a cached rejection;
+      // the next command call retries openExecution instead.
+      opened.catch(() => {
+        if (executionPromise === opened) {
+          executionPromise = undefined;
+        }
+      });
+      executionPromise = opened;
+    }
+    return executionPromise;
+  };
+  const closeExecution = async (reason: string) => {
+    const current = executionPromise;
+    executionPromise = undefined;
+    if (current) {
+      await (await current).close(reason);
+    }
+  };
+
+  api.registerNodeHostCommand({
+    command: "screen.snapshot",
+    cap: "screen",
+    dangerous: false,
+    isAvailable: () => provider.isAvailable(),
+    watchAvailability: (context, onChange) => {
+      const stopWatching = provider.watchAvailability?.(context, onChange);
+      return () => {
+        stopWatching?.();
+        void closeExecution("node-host-stop");
+      };
+    },
+    handle: async (paramsJSON, _io, context) =>
+      await (await getExecution(context)).snapshot(paramsJSON, context?.signal),
+  });
+  api.registerNodeHostCommand({
+    command: "computer.act",
+    cap: "computer",
+    dangerous: true,
+    isAvailable: () => provider.isAvailable(),
+    handle: async (paramsJSON, _io, context) =>
+      await (await getExecution(context)).act(paramsJSON, context?.signal),
+  });
+  // Preserve the existing dangerous-command policy: allowlisting happens
+  // first, then this final Gateway guard forwards the armed invocation.
+  api.registerNodeInvokePolicy({
+    commands: ["computer.act"],
+    dangerous: true,
+    handle: async (context) => await context.invokeNode(),
+  });
+}