From 848a7e30b39f2dc1dc55183c6cd4176a9017471a Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 13 Aug 2026 23:32:32 -0700 Subject: [PATCH] refactor(computer-use): one canonical wire contract + node-host provider seam (#123509) * refactor(computer-use): add provider seam * refactor(computer-use): retry provider open after failure; drop changelog entry --- docs/plugins/sdk-entrypoints.md | 16 ++ docs/plugins/sdk-overview.md | 5 + extensions/cua-computer/index.ts | 16 +- extensions/cua-computer/src/actions.ts | 36 ---- extensions/cua-computer/src/commands.test.ts | 58 +++-- extensions/cua-computer/src/commands.ts | 199 ++++++++--------- package.json | 4 + scripts/lib/plugin-sdk-entrypoints.json | 1 + scripts/plugin-sdk-surface-report.mts | 9 +- src/agents/tools/computer-tool.ts | 58 +++-- src/cli/nodes-screen.ts | 31 +-- src/plugin-sdk/computer-use.ts | 15 ++ src/plugins/computer-use-contract.test.ts | 102 +++++++++ src/plugins/computer-use-contract.ts | 215 +++++++++++++++++++ 14 files changed, 516 insertions(+), 249 deletions(-) create mode 100644 src/plugin-sdk/computer-use.ts create mode 100644 src/plugins/computer-use-contract.test.ts create mode 100644 src/plugins/computer-use-contract.ts diff --git a/docs/plugins/sdk-entrypoints.md b/docs/plugins/sdk-entrypoints.md index 3aec3d32cd7c..b9f0539e1c8b 100644 --- a/docs/plugins/sdk-entrypoints.md +++ b/docs/plugins/sdk-entrypoints.md @@ -185,6 +185,22 @@ export default definePluginEntry({ startup config; command handlers should still validate availability when invoked. +### Computer Use providers + +**Import:** `openclaw/plugin-sdk/computer-use` + +Node-local Computer Use plugins register one provider through +`registerComputerUseProvider(api, provider)`. The helper owns the +`screen.snapshot` and dangerous `computer.act` command registrations and the +matching Gateway invoke policy; the provider owns availability, execution, +serialization, frame state, driver lifecycle, and cleanup. + +The same entry point exports the canonical TypeBox schemas, static types, and +compiled validators for the two command payloads and the snapshot result. A +node host accepts one provider for the command pair; registering another +provider conflicts with the existing command registration instead of creating +a fallback stack. + ## `defineChannelPluginEntry` **Import:** `openclaw/plugin-sdk/channel-core` diff --git a/docs/plugins/sdk-overview.md b/docs/plugins/sdk-overview.md index 6fdf451ded36..aced7bc0b492 100644 --- a/docs/plugins/sdk-overview.md +++ b/docs/plugins/sdk-overview.md @@ -154,6 +154,11 @@ or fully dynamic tool registration. | `api.registerCommand(def)` | Custom command (bypasses the LLM) | | `api.registerNodeHostCommand(command)` | Command handled by `openclaw node run`; optional `agentTool` metadata can expose it as an agent-visible tool while the node is connected | +Computer Use providers use `registerComputerUseProvider(api, provider)` from +`openclaw/plugin-sdk/computer-use`. It registers the shared +`screen.snapshot`/`computer.act` node-host envelope once while the provider +keeps its driver, frame, availability, and execution lifecycle local. + Plugin commands can set `agentPromptGuidance` when the agent needs a short, command-owned routing hint. Keep that text about the command itself; do not add provider- or plugin-specific policy to core prompt builders. diff --git a/extensions/cua-computer/index.ts b/extensions/cua-computer/index.ts index 156f9e6dd723..a7e0980d5d51 100644 --- a/extensions/cua-computer/index.ts +++ b/extensions/cua-computer/index.ts @@ -1,6 +1,7 @@ +import { registerComputerUseProvider } from "openclaw/plugin-sdk/computer-use"; import { buildPluginConfigSchema, definePluginEntry } from "openclaw/plugin-sdk/plugin-entry"; import { z } from "zod"; -import { createCuaComputerCommands } from "./src/commands.js"; +import { createCuaComputerProvider } from "./src/commands.js"; const CuaComputerConfigSchema = z.strictObject({ // Keep the shipped daemon setting as a named no-op: strict validation accepts @@ -22,17 +23,6 @@ export default definePluginEntry({ `Invalid cua-computer plugin config: ${parsed.error.issues[0]?.message ?? "invalid config"}`, ); } - for (const command of createCuaComputerCommands()) { - api.registerNodeHostCommand(command); - } - // computer.act is dangerous-by-default and therefore also requires the - // operator's explicit gateway.nodes.commands.allow entry. The plugin - // policy is the final Gateway guard and the only path that may forward the - // already-allowlisted invocation to the paired node. - api.registerNodeInvokePolicy({ - commands: ["computer.act"], - dangerous: true, - handle: async (ctx) => await ctx.invokeNode(), - }); + registerComputerUseProvider(api, createCuaComputerProvider()); }, }); diff --git a/extensions/cua-computer/src/actions.ts b/extensions/cua-computer/src/actions.ts index e8ab3f8b4230..d4136550d361 100644 --- a/extensions/cua-computer/src/actions.ts +++ b/extensions/cua-computer/src/actions.ts @@ -1,41 +1,5 @@ -import { z } from "zod"; import type { CuaLastFrame } from "./frame.js"; -const COMPUTER_ACTIONS = [ - "left_click", - "right_click", - "middle_click", - "double_click", - "triple_click", - "mouse_move", - "left_click_drag", - "left_mouse_down", - "left_mouse_up", - "scroll", - "type", - "key", - "hold_key", -] as const; - -export const ComputerActParamsSchema = z.strictObject({ - action: z.enum(COMPUTER_ACTIONS), - displayFrameId: z.string().optional(), - x: z.number().finite().nonnegative().optional(), - y: z.number().finite().nonnegative().optional(), - fromX: z.number().finite().nonnegative().optional(), - fromY: z.number().finite().nonnegative().optional(), - text: z.string().optional(), - keys: z.string().optional(), - modifiers: z.string().optional(), - scrollDirection: z.enum(["up", "down", "left", "right"]).optional(), - scrollAmount: z.number().int().positive().optional(), - durationMs: z.number().int().nonnegative().optional(), - screenIndex: z.number().int().nonnegative().optional(), - refWidth: z.number().int().positive().optional(), -}); - -export type ComputerActParams = z.infer; - const MODIFIER_ALIASES = new Map([ ["ctrl", "ctrl"], ["control", "ctrl"], diff --git a/extensions/cua-computer/src/commands.test.ts b/extensions/cua-computer/src/commands.test.ts index 90db04164088..162c4907d8d6 100644 --- a/extensions/cua-computer/src/commands.test.ts +++ b/extensions/cua-computer/src/commands.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from "vitest"; -import { createCuaComputerCommands } from "./commands.js"; +import { createCuaComputerProvider } from "./commands.js"; import { ClickButton, ScrollDirection, @@ -68,25 +68,25 @@ function driver() { }; } -function commands(session: CuaDriverSession) { - return createCuaComputerCommands({ +async function execution(session: CuaDriverSession) { + return await createCuaComputerProvider({ platform: "linux", driver: session, imageProcessor: { encode: vi.fn(async () => ({ data: Buffer.from("jpeg"), width: 100, height: 50 })), }, - }); + }).openExecution({}); } -describe("cua-computer direct SDK commands", () => { +describe("cua-computer provider", () => { it("uses one typed session for snapshot and frame-authorized click", async () => { const { session, getDesktopState, getScreenSize, click } = driver(); - const [snapshot, act] = commands(session); - const screen = JSON.parse(await snapshot!.handle('{"format":"png","maxWidth":100}')) as { + const computer = await execution(session); + const screen = JSON.parse(await computer.snapshot('{"format":"png","maxWidth":100}')) as { displayFrameId: string; width: number; }; - await act!.handle( + await computer.act( JSON.stringify({ action: "left_click", displayFrameId: screen.displayFrameId, @@ -110,9 +110,9 @@ describe("cua-computer direct SDK commands", () => { it("maps scroll and key through typed SDK enums", async () => { const { session, typeText, pressKey } = driver(); - const [, act] = commands(session); - await act!.handle('{"action":"type","text":"hello"}'); - await act!.handle('{"action":"key","keys":"ctrl+enter"}'); + const computer = await execution(session); + await computer.act('{"action":"type","text":"hello"}'); + await computer.act('{"action":"key","keys":"ctrl+enter"}'); expect(typeText).toHaveBeenCalledWith("hello", undefined); expect(pressKey).toHaveBeenCalledWith({ key: "enter", modifiers: ["ctrl"] }, undefined); expect(ScrollDirection.Down).toBeTypeOf("number"); @@ -120,14 +120,14 @@ describe("cua-computer direct SDK commands", () => { it("maps all remaining projected desktop actions through direct SDK methods", async () => { const { session, scroll, moveCursor, drag } = driver(); - const [snapshot, act] = commands(session); - const screen = JSON.parse(await snapshot!.handle('{"format":"png","maxWidth":100}')) as { + const computer = await execution(session); + const screen = JSON.parse(await computer.snapshot('{"format":"png","maxWidth":100}')) as { displayFrameId: string; width: number; }; const frame = { displayFrameId: screen.displayFrameId, refWidth: screen.width }; - await act!.handle( + await computer.act( JSON.stringify({ action: "scroll", ...frame, @@ -137,8 +137,8 @@ describe("cua-computer direct SDK commands", () => { scrollAmount: 4, }), ); - await act!.handle(JSON.stringify({ action: "mouse_move", ...frame, x: 11, y: 21 })); - await act!.handle( + await computer.act(JSON.stringify({ action: "mouse_move", ...frame, x: 11, y: 21 })); + await computer.act( JSON.stringify({ action: "left_click_drag", ...frame, @@ -169,13 +169,13 @@ describe("cua-computer direct SDK commands", () => { errorCode: "desktop_unavailable", text: "desktop input is unavailable", }); - const [snapshot, act] = commands(session); - const screen = JSON.parse(await snapshot!.handle('{"format":"png","maxWidth":100}')) as { + const computer = await execution(session); + const screen = JSON.parse(await computer.snapshot('{"format":"png","maxWidth":100}')) as { displayFrameId: string; width: number; }; await expect( - act!.handle( + computer.act( JSON.stringify({ action: "left_click", displayFrameId: screen.displayFrameId, @@ -189,14 +189,14 @@ describe("cua-computer direct SDK commands", () => { it("rejects a mismatched reference width before desktop input", async () => { const { session, click } = driver(); - const [snapshot, act] = commands(session); - const screen = JSON.parse(await snapshot!.handle('{"format":"png","maxWidth":100}')) as { + const computer = await execution(session); + const screen = JSON.parse(await computer.snapshot('{"format":"png","maxWidth":100}')) as { displayFrameId: string; width: number; }; await expect( - act!.handle( + computer.act( JSON.stringify({ action: "left_click", displayFrameId: screen.displayFrameId, @@ -213,7 +213,7 @@ describe("cua-computer direct SDK commands", () => { const { session, dispose } = driver(); const createDriver = vi.fn(() => session); const clearInterval = vi.fn(); - const [snapshot] = createCuaComputerCommands({ + const provider = createCuaComputerProvider({ platform: "linux", createDriver, imageProcessor: { @@ -224,10 +224,11 @@ describe("cua-computer direct SDK commands", () => { }); expect(createDriver).not.toHaveBeenCalled(); - await snapshot!.handle('{"format":"png","maxWidth":100}'); + const computer = await provider.openExecution({}); + await computer.snapshot('{"format":"png","maxWidth":100}'); expect(createDriver).toHaveBeenCalledOnce(); - const stop = snapshot!.watchAvailability?.({ config: {} as never, env: {} }, vi.fn()); + const stop = provider.watchAvailability?.({ config: {} as never, env: {} }, vi.fn()); stop?.(); await Promise.resolve(); expect(clearInterval).toHaveBeenCalledOnce(); @@ -236,12 +237,9 @@ describe("cua-computer direct SDK commands", () => { it("passes node invocation cancellation to the direct SDK", async () => { const { session, getDesktopState } = driver(); - const [snapshot] = commands(session); + const computer = await execution(session); const signal = AbortSignal.abort(); - await snapshot!.handle('{"format":"png","maxWidth":100}', undefined, { - sendNodeEvent: vi.fn(), - signal, - }); + await computer.snapshot('{"format":"png","maxWidth":100}', signal); expect(getDesktopState).toHaveBeenCalledWith(signal); }); }); diff --git a/extensions/cua-computer/src/commands.ts b/extensions/cua-computer/src/commands.ts index 082116d4d8cf..211c54454c6b 100644 --- a/extensions/cua-computer/src/commands.ts +++ b/extensions/cua-computer/src/commands.ts @@ -1,17 +1,16 @@ import fs from "node:fs"; import path from "node:path"; +import { + parseComputerActParamsJSON, + parseScreenSnapshotParamsJSON, + type ComputerActParams, + type ComputerUseProvider, +} from "openclaw/plugin-sdk/computer-use"; import { canonicalizeBase64 } from "openclaw/plugin-sdk/media-runtime"; -import type { OpenClawPluginNodeHostCommand } from "openclaw/plugin-sdk/plugin-entry"; import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path"; import { createRastermill } from "rastermill"; import { z } from "zod"; -import { - ComputerActParamsSchema, - normalizeModifiers, - parseKeyChord, - scalePoint, - type ComputerActParams, -} from "./actions.js"; +import { normalizeModifiers, parseKeyChord, scalePoint } from "./actions.js"; import { ClickButton, ScrollDirection, @@ -35,13 +34,6 @@ const AVAILABILITY_POLL_MS = 5_000; // display; budget above it so full-resolution snapshots reach the downscaler. const MAX_IMAGE_PIXELS = 40_000_000; -const SnapshotParamsSchema = z.strictObject({ - screenIndex: z.number().int().nonnegative().optional(), - maxWidth: z.number().int().positive().optional(), - quality: z.number().finite().optional(), - format: z.enum(["jpeg", "png"]).optional(), -}); - const DesktopStateSchema = z.object({ platform: z.string().min(1), display: z.string().min(1), @@ -69,7 +61,7 @@ type ImageProcessor = { ): Promise<{ data: Buffer; width: number; height: number }>; }; -type CuaComputerCommandsOptions = { +type CuaComputerProviderOptions = { platform?: NodeJS.Platform; env?: NodeJS.ProcessEnv; driver?: CuaDriverSession; @@ -97,22 +89,6 @@ class PromiseQueue { } } -function parseParams(schema: z.ZodType, paramsJSON: string | null | undefined): T { - let value: unknown; - try { - value = JSON.parse(paramsJSON ?? "{}"); - } catch { - throw new Error("COMPUTER_INVALID_REQUEST: params must be valid JSON"); - } - const parsed = schema.safeParse(value); - if (!parsed.success) { - throw new Error( - `COMPUTER_INVALID_REQUEST: ${parsed.error.issues[0]?.message ?? "invalid params"}`, - ); - } - return parsed.data; -} - function assertPrimaryDisplay(screenIndex: number | undefined): void { if (screenIndex !== undefined && screenIndex !== 0) { throw new Error( @@ -407,9 +383,9 @@ async function handleAct( return JSON.stringify({ ok: true }); } -export function createCuaComputerCommands( - options: CuaComputerCommandsOptions = {}, -): OpenClawPluginNodeHostCommand[] { +export function createCuaComputerProvider( + options: CuaComputerProviderOptions = {}, +): ComputerUseProvider { const platform = options.platform ?? process.platform; const env = options.env ?? process.env; let ownedDriver: CuaDriverSession | undefined; @@ -429,17 +405,14 @@ export function createCuaComputerCommands( await current?.dispose(); }; const imageProcessor = options.imageProcessor ?? createImageProcessor(env); - const queue = new PromiseQueue(); - const frameState: CuaFrameState = { generation: "uninitialized" }; const interval = options.setInterval ?? setInterval; const clear = options.clearInterval ?? clearInterval; const isSupportedPlatform = platform === "linux" || platform === "win32"; const isAvailable = () => isSupportedPlatform && driver().isAvailable(); - const snapshot: OpenClawPluginNodeHostCommand = { - command: "screen.snapshot", - cap: "screen", - dangerous: false, + return { + id: "cua-computer", + label: "CUA Computer", isAvailable, watchAvailability: (_context, onChange) => { let knownAvailable = isAvailable(); @@ -457,76 +430,78 @@ export function createCuaComputerCommands( void disposeOwnedDriver(); }; }, - handle: async (paramsJSON, _io, context) => - await queue.run(async () => { - if (!isSupportedPlatform) { - throw new Error("COMPUTER_DRIVER_UNAVAILABLE: cua-computer supports Windows and Linux"); - } - const params = parseParams(SnapshotParamsSchema, paramsJSON); - assertPrimaryDisplay(params.screenIndex); - const format = params.format ?? "jpeg"; - const maxWidth = params.maxWidth ?? (format === "png" ? 900 : 1_600); - const quality = Math.min(1, Math.max(0.05, params.quality ?? 0.72)); - const desktop = await driver().getDesktopState(context?.signal); - const geometry = desktopGeometry(desktop); - // cua-driver desktop input consumes native get_desktop_state PNG pixels, - // and on every supported backend the driver reports screen geometry in - // that same physical-pixel space (Windows PMv2, Linux X11/Wayland). If a - // capture ever diverges from screen geometry, our screenshot->native - // scaling would mis-target input, so refuse rather than click blind. - if ( - geometry.screenWidth !== geometry.screenshotWidth || - geometry.screenHeight !== geometry.screenshotHeight - ) { - throw new Error( - "COMPUTER_UNSUPPORTED_DISPLAY: cua-driver reported capture and screen geometry in different pixel spaces", - ); - } - const nativePng = desktopPng(desktop); - let encoded = nativePng; - let width = geometry.screenshotWidth; - let height = geometry.screenshotHeight; - if (format === "jpeg" || width > maxWidth) { - const result = await imageProcessor.encode(nativePng, { - format, - ...(format === "jpeg" ? { quality: Math.round(quality * 100) } : {}), - ...(width > maxWidth ? { resize: { width: maxWidth, enlarge: false } } : {}), - }); - encoded = result.data; - width = result.width; - height = result.height; - } - frameState.generation = driver().generation; - const displayFrameId = issueFrame(frameState, geometry, { width, height }); - return JSON.stringify({ - format, - base64: encoded.toString("base64"), - displayFrameId, - screenIndex: 0, - width, - height, - }); - }), + openExecution: async () => { + const queue = new PromiseQueue(); + const frameState: CuaFrameState = { generation: "uninitialized" }; + return { + snapshot: async (paramsJSON, signal) => + await queue.run(async () => { + if (!isSupportedPlatform) { + throw new Error( + "COMPUTER_DRIVER_UNAVAILABLE: cua-computer supports Windows and Linux", + ); + } + const params = parseScreenSnapshotParamsJSON(paramsJSON); + assertPrimaryDisplay(params.screenIndex); + const format = params.format ?? "jpeg"; + const maxWidth = params.maxWidth ?? (format === "png" ? 900 : 1_600); + const quality = Math.min(1, Math.max(0.05, params.quality ?? 0.72)); + const desktop = await driver().getDesktopState(signal); + const geometry = desktopGeometry(desktop); + // cua-driver desktop input consumes native get_desktop_state PNG pixels, + // and on every supported backend the driver reports screen geometry in + // that same physical-pixel space (Windows PMv2, Linux X11/Wayland). If a + // capture ever diverges from screen geometry, our screenshot->native + // scaling would mis-target input, so refuse rather than click blind. + if ( + geometry.screenWidth !== geometry.screenshotWidth || + geometry.screenHeight !== geometry.screenshotHeight + ) { + throw new Error( + "COMPUTER_UNSUPPORTED_DISPLAY: cua-driver reported capture and screen geometry in different pixel spaces", + ); + } + const nativePng = desktopPng(desktop); + let encoded = nativePng; + let width = geometry.screenshotWidth; + let height = geometry.screenshotHeight; + if (format === "jpeg" || width > maxWidth) { + const result = await imageProcessor.encode(nativePng, { + format, + ...(format === "jpeg" ? { quality: Math.round(quality * 100) } : {}), + ...(width > maxWidth ? { resize: { width: maxWidth, enlarge: false } } : {}), + }); + encoded = result.data; + width = result.width; + height = result.height; + } + frameState.generation = driver().generation; + const displayFrameId = issueFrame(frameState, geometry, { width, height }); + return JSON.stringify({ + format, + base64: encoded.toString("base64"), + displayFrameId, + screenIndex: 0, + width, + height, + }); + }), + act: async (paramsJSON, signal) => + await queue.run(async () => { + if (!isSupportedPlatform) { + throw new Error( + "COMPUTER_DRIVER_UNAVAILABLE: cua-computer supports Windows and Linux", + ); + } + return await handleAct( + driver(), + frameState, + parseComputerActParamsJSON(paramsJSON), + signal, + ); + }), + close: async () => await disposeOwnedDriver(), + }; + }, }; - - const act: OpenClawPluginNodeHostCommand = { - command: "computer.act", - cap: "computer", - dangerous: true, - isAvailable, - handle: async (paramsJSON, _io, context) => - await queue.run(async () => { - if (!isSupportedPlatform) { - throw new Error("COMPUTER_DRIVER_UNAVAILABLE: cua-computer supports Windows and Linux"); - } - return await handleAct( - driver(), - frameState, - parseParams(ComputerActParamsSchema, paramsJSON), - context?.signal, - ); - }), - }; - - return [snapshot, act]; } diff --git a/package.json b/package.json index c03f38e2c751..a6edd1acfc63 100644 --- a/package.json +++ b/package.json @@ -1065,6 +1065,10 @@ "./plugin-sdk/node-host": { "default": "./dist/plugin-sdk/node-host.js" }, + "./plugin-sdk/computer-use": { + "types": "./dist/plugin-sdk/computer-use.d.ts", + "default": "./dist/plugin-sdk/computer-use.js" + }, "./plugin-sdk/response-limit-runtime": { "default": "./dist/plugin-sdk/response-limit-runtime.js" }, diff --git a/scripts/lib/plugin-sdk-entrypoints.json b/scripts/lib/plugin-sdk-entrypoints.json index 3393dd513a17..e06b3c31ff5c 100644 --- a/scripts/lib/plugin-sdk-entrypoints.json +++ b/scripts/lib/plugin-sdk-entrypoints.json @@ -209,6 +209,7 @@ "runtime-fetch", "inline-image-data-url-runtime", "node-host", + "computer-use", "response-limit-runtime", "session-binding-runtime", "session-catalog", diff --git a/scripts/plugin-sdk-surface-report.mts b/scripts/plugin-sdk-surface-report.mts index 6c12bd86294b..2b923780a613 100644 --- a/scripts/plugin-sdk-surface-report.mts +++ b/scripts/plugin-sdk-surface-report.mts @@ -189,7 +189,8 @@ export function readPluginSdkSurfaceBudgets(env: NodeJS.ProcessEnv = process.env // +1: dependency-light channel streaming config readers for doctor closures // (realtime-voice-activation is private-local and not counted here). // +1: registry-bound plugin command planning and exact selected execution. - 144, + // +1: canonical Computer Use wire contract and node-host provider seam. + 145, env, ), publicExports: readPluginSdkSurfaceBudgetEnv( @@ -272,7 +273,8 @@ export function readPluginSdkSurfaceBudgets(env: NodeJS.ProcessEnv = process.env // +1: normalized Gateway public origin resolver for plugin-generated links. // -2: retire the dead progress-draft render reader; it counted twice via // channel-outbound and channel-message's wildcard re-export of it. - 4306, + // +11: Computer Use schemas/types plus parsers, compiler, and provider registration. + 4317, env, ), publicFunctionExports: readPluginSdkSurfaceBudgetEnv( @@ -341,7 +343,8 @@ export function readPluginSdkSurfaceBudgets(env: NodeJS.ProcessEnv = process.env // +1: normalized Gateway public origin resolver for plugin-generated links. // -2: retire the dead progress-draft render reader; it counted twice via // channel-outbound and channel-message's wildcard re-export of it. - 2570, + // +4: Computer Use wire parsers, validator compiler, and provider registration. + 2574, env, ), publicDeprecatedExports: readPluginSdkSurfaceBudgetEnv( diff --git a/src/agents/tools/computer-tool.ts b/src/agents/tools/computer-tool.ts index 35cb2a945978..9cf18671c704 100644 --- a/src/agents/tools/computer-tool.ts +++ b/src/agents/tools/computer-tool.ts @@ -14,6 +14,10 @@ import { Type } from "typebox"; import { parseScreenSnapshotPayload } from "../../cli/nodes-screen.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; import { formatErrorMessage } from "../../infra/errors.js"; +import type { + ComputerActParams, + ScreenSnapshotParams, +} from "../../plugins/computer-use-contract.js"; import { sleep } from "../../utils/sleep.js"; import { DEFAULT_IMAGE_MAX_DIMENSION_PX, @@ -77,7 +81,7 @@ const COMPUTER_TOOL_ACTIONS = [ type ComputerToolAction = (typeof COMPUTER_TOOL_ACTIONS)[number]; -const INPUT_ACTIONS = new Set([ +const INPUT_ACTIONS = new Set([ "left_click", "right_click", "middle_click", @@ -93,6 +97,10 @@ const INPUT_ACTIONS = new Set([ "hold_key", ]); +function isComputerActAction(action: ComputerToolAction): action is ComputerActParams["action"] { + return INPUT_ACTIONS.has(action as ComputerActParams["action"]); +} + const COORDINATE_REQUIRED_ACTIONS = new Set([ "left_click", "right_click", @@ -126,6 +134,12 @@ const MODIFIER_TEXT_ACTIONS = new Set([ const SCROLL_DIRECTIONS = ["up", "down", "left", "right"] as const; +function isScrollDirection( + value: string, +): value is NonNullable { + return SCROLL_DIRECTIONS.some((direction) => direction === value); +} + const ComputerToolSchema = Type.Object({ action: stringEnum(COMPUTER_TOOL_ACTIONS), ...gatewayCallOptionSchemaProperties(), @@ -177,23 +191,6 @@ const ComputerToolSchema = Type.Object({ ), }); -type ComputerActWireParams = { - action: string; - displayFrameId?: string; - x?: number; - y?: number; - fromX?: number; - fromY?: number; - text?: string; - keys?: string; - modifiers?: string; - scrollDirection?: string; - scrollAmount?: number; - durationMs?: number; - screenIndex?: number; - refWidth: number; -}; - function readCoordinate( params: Record, key: "coordinate" | "startCoordinate", @@ -236,14 +233,14 @@ function readModifiers(params: Record, action: ComputerToolActi /** Builds the computer.act wire params for one tool input action. */ function buildComputerActParams(params: { - action: ComputerToolAction; + action: ComputerActParams["action"]; input: Record; screenIndex: number; displayFrameId?: string; refWidth?: number; -}): ComputerActWireParams { +}): ComputerActParams { const { action, input } = params; - const wire: ComputerActWireParams = { + const wire: ComputerActParams = { action, screenIndex: params.screenIndex, refWidth: params.refWidth ?? COMPUTER_REF_WIDTH, @@ -278,7 +275,7 @@ function buildComputerActParams(params: { } case "scroll": { const direction = normalizeOptionalLowercaseString(input.scrollDirection); - if (!direction || !SCROLL_DIRECTIONS.includes(direction as never)) { + if (!direction || !isScrollDirection(direction)) { throw new Error("scrollDirection up|down|left|right required for scroll"); } wire.scrollDirection = direction; @@ -409,16 +406,17 @@ async function captureScreenshot(params: { refWidth: number; signal?: AbortSignal; }): Promise { + const commandParams: ScreenSnapshotParams = { + screenIndex: params.screenIndex, + maxWidth: params.refWidth, + quality: SCREENSHOT_QUALITY, + format: "jpeg", + }; const payload = await invokeNodeCommand({ gatewayOpts: params.gatewayOpts, nodeId: params.nodeId, command: SCREEN_SNAPSHOT_COMMAND, - commandParams: { - screenIndex: params.screenIndex, - maxWidth: params.refWidth, - quality: SCREENSHOT_QUALITY, - format: "jpeg", - }, + commandParams, signal: params.signal, }); const parsed = parseScreenSnapshotPayload(payload); @@ -861,8 +859,8 @@ export function createComputerTool(options?: { break; } - if (!INPUT_ACTIONS.has(action)) { - throw new Error(`Unknown action: ${action}`); + if (!isComputerActAction(action)) { + throw new Error(`Unknown action: ${String(action)}`); } const wireParams = buildComputerActParams({ action, diff --git a/src/cli/nodes-screen.ts b/src/cli/nodes-screen.ts index 7efdb895a65b..e2b4436c283d 100644 --- a/src/cli/nodes-screen.ts +++ b/src/cli/nodes-screen.ts @@ -1,6 +1,10 @@ // Screen-recording payload helpers for node media commands. import * as path from "node:path"; import { extnameFromAnyPath } from "@openclaw/media-core/file-name"; +import { + parseScreenSnapshotResult, + type ScreenSnapshotResult, +} from "../plugins/computer-use-contract.js"; import { writeBase64ToFile } from "./nodes-camera.js"; import { asRecord, readStringValue, resolveTempPathParts } from "./nodes-media-utils.js"; @@ -48,32 +52,9 @@ export async function writeScreenRecordToFile( } /** Validated payload returned by `nodes screen snapshot` RPC calls. */ -type ScreenSnapshotPayload = { - format: string; - base64: string; - /** Node-issued token binding this image to one physical display geometry. */ - displayFrameId?: string; - screenIndex?: number; - width?: number; - height?: number; -}; - /** Validate and normalize an unknown screen-snapshot payload. */ -export function parseScreenSnapshotPayload(value: unknown): ScreenSnapshotPayload { - const obj = asRecord(value); - const format = readStringValue(obj.format); - const base64 = readStringValue(obj.base64); - if (!format || !base64) { - throw new Error("invalid screen.snapshot payload"); - } - return { - format, - base64, - displayFrameId: readStringValue(obj.displayFrameId) || undefined, - screenIndex: typeof obj.screenIndex === "number" ? obj.screenIndex : undefined, - width: typeof obj.width === "number" ? obj.width : undefined, - height: typeof obj.height === "number" ? obj.height : undefined, - }; +export function parseScreenSnapshotPayload(value: unknown): ScreenSnapshotResult { + return parseScreenSnapshotResult(value); } /** diff --git a/src/plugin-sdk/computer-use.ts b/src/plugin-sdk/computer-use.ts new file mode 100644 index 000000000000..68f1955db73c --- /dev/null +++ b/src/plugin-sdk/computer-use.ts @@ -0,0 +1,15 @@ +export { + ComputerActParamsSchema, + ScreenSnapshotParamsSchema, + ScreenSnapshotResultSchema, + compileComputerUseValidator, + parseComputerActParamsJSON, + parseScreenSnapshotParamsJSON, + registerComputerUseProvider, +} from "../plugins/computer-use-contract.js"; +export type { + ComputerActParams, + ComputerUseProvider, + ScreenSnapshotParams, + ScreenSnapshotResult, +} from "../plugins/computer-use-contract.js"; diff --git a/src/plugins/computer-use-contract.test.ts b/src/plugins/computer-use-contract.test.ts new file mode 100644 index 000000000000..d9fe2ffcd2eb --- /dev/null +++ b/src/plugins/computer-use-contract.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, it, vi } from "vitest"; +import { + parseComputerActParamsJSON, + parseScreenSnapshotResult, + registerComputerUseProvider, + type ComputerUseProvider, +} from "./computer-use-contract.js"; +import type { OpenClawPluginNodeHostCommand, OpenClawPluginNodeInvokePolicy } from "./types.js"; + +describe("Computer Use wire contract", () => { + it("validates the canonical computer.act payload", () => { + expect( + parseComputerActParamsJSON( + JSON.stringify({ + action: "left_click", + displayFrameId: "frame-1", + x: 10, + y: 20, + refWidth: 1280, + }), + ), + ).toEqual({ + action: "left_click", + displayFrameId: "frame-1", + x: 10, + y: 20, + refWidth: 1280, + }); + expect(() => parseComputerActParamsJSON('{"action":"left_click","unexpected":true}')).toThrow( + "COMPUTER_INVALID_REQUEST", + ); + }); + + it("projects the canonical screen.snapshot result", () => { + expect( + parseScreenSnapshotResult({ + format: "jpeg", + base64: "aGk=", + displayFrameId: "frame-1", + width: 100, + height: 50, + capturedAtMs: 42, + ignored: true, + }), + ).toEqual({ + format: "jpeg", + base64: "aGk=", + displayFrameId: "frame-1", + width: 100, + height: 50, + capturedAtMs: 42, + }); + }); +}); + +describe("Computer Use provider registration", () => { + it("registers one command pair and dispatches both through one execution", async () => { + const commands: OpenClawPluginNodeHostCommand[] = []; + const policies: OpenClawPluginNodeInvokePolicy[] = []; + const snapshot = vi.fn(async () => "snapshot"); + const act = vi.fn(async () => "act"); + const close = vi.fn(async () => {}); + const stopWatching = vi.fn(); + const openExecution = vi.fn(async () => ({ snapshot, act, close })); + const provider: ComputerUseProvider = { + id: "fixture", + label: "Fixture", + isAvailable: () => true, + watchAvailability: () => stopWatching, + openExecution, + }; + + registerComputerUseProvider( + { + registerNodeHostCommand: (command) => commands.push(command), + registerNodeInvokePolicy: (policy) => policies.push(policy), + }, + provider, + ); + + expect(commands.map(({ command, cap, dangerous }) => ({ command, cap, dangerous }))).toEqual([ + { command: "screen.snapshot", cap: "screen", dangerous: false }, + { command: "computer.act", cap: "computer", dangerous: true }, + ]); + expect(policies).toHaveLength(1); + expect(policies[0]).toMatchObject({ commands: ["computer.act"], dangerous: true }); + + const signal = new AbortController().signal; + const context = { sendNodeEvent: vi.fn(), sessionKey: "session-1", signal }; + await expect(commands[0]!.handle("{}", undefined, context)).resolves.toBe("snapshot"); + await expect(commands[1]!.handle("{}", undefined, context)).resolves.toBe("act"); + expect(openExecution).toHaveBeenCalledOnce(); + expect(openExecution).toHaveBeenCalledWith({ sessionKey: "session-1" }); + expect(snapshot).toHaveBeenCalledWith("{}", signal); + expect(act).toHaveBeenCalledWith("{}", signal); + + const stop = commands[0]!.watchAvailability?.({ config: {} as never, env: {} }, vi.fn()); + stop?.(); + await vi.waitFor(() => expect(close).toHaveBeenCalledWith("node-host-stop")); + expect(stopWatching).toHaveBeenCalledOnce(); + }); +}); diff --git a/src/plugins/computer-use-contract.ts b/src/plugins/computer-use-contract.ts new file mode 100644 index 000000000000..19690c122c7b --- /dev/null +++ b/src/plugins/computer-use-contract.ts @@ -0,0 +1,215 @@ +import { type Static, type TSchema, Type } from "typebox"; +import { Compile } from "typebox/compile"; +import type { OpenClawPluginApi } from "./plugin-api.types.js"; +import type { + OpenClawPluginNodeHostCommandAvailabilityContext, + OpenClawPluginNodeHostCommandContext, +} from "./types.node-host.js"; + +const COMPUTER_ACT_ACTIONS = [ + "left_click", + "right_click", + "middle_click", + "double_click", + "triple_click", + "mouse_move", + "left_click_drag", + "left_mouse_down", + "left_mouse_up", + "scroll", + "type", + "key", + "hold_key", +] as const; + +const SCROLL_DIRECTIONS = ["up", "down", "left", "right"] as const; + +/** Canonical inner payload accepted by the `computer.act` node command. */ +export const ComputerActParamsSchema = Type.Object( + { + action: Type.Enum(COMPUTER_ACT_ACTIONS, { type: "string" }), + displayFrameId: Type.Optional(Type.String()), + x: Type.Optional(Type.Number({ minimum: 0 })), + y: Type.Optional(Type.Number({ minimum: 0 })), + fromX: Type.Optional(Type.Number({ minimum: 0 })), + fromY: Type.Optional(Type.Number({ minimum: 0 })), + text: Type.Optional(Type.String()), + keys: Type.Optional(Type.String()), + modifiers: Type.Optional(Type.String()), + scrollDirection: Type.Optional(Type.Enum(SCROLL_DIRECTIONS, { type: "string" })), + scrollAmount: Type.Optional(Type.Integer({ minimum: 1 })), + durationMs: Type.Optional(Type.Integer({ minimum: 0 })), + screenIndex: Type.Optional(Type.Integer({ minimum: 0 })), + refWidth: Type.Optional(Type.Integer({ minimum: 1 })), + }, + { additionalProperties: false }, +); + +/** Canonical inner payload accepted by the `screen.snapshot` node command. */ +export const ScreenSnapshotParamsSchema = Type.Object( + { + screenIndex: Type.Optional(Type.Integer({ minimum: 0 })), + maxWidth: Type.Optional(Type.Integer({ minimum: 1 })), + quality: Type.Optional(Type.Number()), + format: Type.Optional(Type.Enum(["jpeg", "png"], { type: "string" })), + }, + { additionalProperties: false }, +); + +/** Canonical inner payload returned by the `screen.snapshot` node command. */ +export const ScreenSnapshotResultSchema = Type.Object({ + format: Type.Enum(["jpeg", "png"], { type: "string" }), + base64: Type.String({ minLength: 1 }), + displayFrameId: Type.Optional(Type.String()), + screenIndex: Type.Optional(Type.Number()), + width: Type.Optional(Type.Number()), + height: Type.Optional(Type.Number()), + capturedAtMs: Type.Optional(Type.Integer({ minimum: 0 })), +}); + +export type ComputerActParams = Static; +export type ScreenSnapshotParams = Static; +export type ScreenSnapshotResult = Static; + +type ComputerUseValidator = (value: unknown) => value is Value; + +/** Compile one Computer Use wire schema into a reusable type-guard validator. */ +export function compileComputerUseValidator( + schema: Schema, +): ComputerUseValidator> { + const validator = Compile(schema); + return (value: unknown): value is Static => validator.Check(value); +} + +const validateComputerActParams = compileComputerUseValidator(ComputerActParamsSchema); +const validateScreenSnapshotParams = compileComputerUseValidator(ScreenSnapshotParamsSchema); +const validateScreenSnapshotResult = compileComputerUseValidator(ScreenSnapshotResultSchema); + +function parseParamsJSON( + paramsJSON: string | null | undefined, + validate: ComputerUseValidator, +): Value { + let value: unknown; + try { + value = JSON.parse(paramsJSON ?? "{}"); + } catch { + throw new Error("COMPUTER_INVALID_REQUEST: params must be valid JSON"); + } + if (!validate(value)) { + throw new Error("COMPUTER_INVALID_REQUEST: invalid params"); + } + return value; +} + +export function parseComputerActParamsJSON( + paramsJSON: string | null | undefined, +): ComputerActParams { + return parseParamsJSON(paramsJSON, validateComputerActParams); +} + +export function parseScreenSnapshotParamsJSON( + paramsJSON: string | null | undefined, +): ScreenSnapshotParams { + return parseParamsJSON(paramsJSON, validateScreenSnapshotParams); +} + +/** Validate and project a `screen.snapshot` result without retaining unknown fields. */ +export function parseScreenSnapshotResult(value: unknown): ScreenSnapshotResult { + if (!validateScreenSnapshotResult(value)) { + throw new Error("invalid screen.snapshot payload"); + } + return { + format: value.format, + base64: value.base64, + ...(value.displayFrameId ? { displayFrameId: value.displayFrameId } : {}), + ...(value.screenIndex !== undefined ? { screenIndex: value.screenIndex } : {}), + ...(value.width !== undefined ? { width: value.width } : {}), + ...(value.height !== undefined ? { height: value.height } : {}), + ...(value.capturedAtMs !== undefined ? { capturedAtMs: value.capturedAtMs } : {}), + }; +} + +type ComputerUseExecution = { + snapshot(paramsJSON: string | null | undefined, signal?: AbortSignal): Promise; + act(paramsJSON: string | null | undefined, signal?: AbortSignal): Promise; + close(reason: string): Promise; +}; + +export type ComputerUseProvider = { + id: string; + label: string; + isAvailable(): boolean; + watchAvailability?: ( + context: OpenClawPluginNodeHostCommandAvailabilityContext, + onChange: () => void, + ) => (() => void) | void; + openExecution(context: { sessionKey?: string }): Promise; +}; + +type ComputerUseRegistrationApi = Pick< + OpenClawPluginApi, + "registerNodeHostCommand" | "registerNodeInvokePolicy" +>; + +/** Register the canonical node-host command pair for one node-local provider. */ +export function registerComputerUseProvider( + api: ComputerUseRegistrationApi, + provider: ComputerUseProvider, +): void { + let executionPromise: Promise | undefined; + + const getExecution = (context?: OpenClawPluginNodeHostCommandContext) => { + if (!executionPromise) { + const opened = provider.openExecution( + context?.sessionKey ? { sessionKey: context.sessionKey } : {}, + ); + // A failed open must not wedge the provider behind a cached rejection; + // the next command call retries openExecution instead. + opened.catch(() => { + if (executionPromise === opened) { + executionPromise = undefined; + } + }); + executionPromise = opened; + } + return executionPromise; + }; + const closeExecution = async (reason: string) => { + const current = executionPromise; + executionPromise = undefined; + if (current) { + await (await current).close(reason); + } + }; + + api.registerNodeHostCommand({ + command: "screen.snapshot", + cap: "screen", + dangerous: false, + isAvailable: () => provider.isAvailable(), + watchAvailability: (context, onChange) => { + const stopWatching = provider.watchAvailability?.(context, onChange); + return () => { + stopWatching?.(); + void closeExecution("node-host-stop"); + }; + }, + handle: async (paramsJSON, _io, context) => + await (await getExecution(context)).snapshot(paramsJSON, context?.signal), + }); + api.registerNodeHostCommand({ + command: "computer.act", + cap: "computer", + dangerous: true, + isAvailable: () => provider.isAvailable(), + handle: async (paramsJSON, _io, context) => + await (await getExecution(context)).act(paramsJSON, context?.signal), + }); + // Preserve the existing dangerous-command policy: allowlisting happens + // first, then this final Gateway guard forwards the armed invocation. + api.registerNodeInvokePolicy({ + commands: ["computer.act"], + dangerous: true, + handle: async (context) => await context.invokeNode(), + }); +}