fix(computer-use): repair artifact verification and post-approval descriptor found by the Linux gate (#124128)

* fix(cua-computer): prove Linux X11 live vertical

* test(computer-use): authenticate isolated Linux rig

* fix(gateway): refresh computer use after node approval

* refactor(cua-computer): resolve the plugin manifest by static import

* fix(gateway): break plugin runtime import cycle

* fix(computer-use): bind live rig to committed helpers
This commit is contained in:
Peter Steinberger
2026-08-15 03:53:23 -07:00
committed by GitHub
parent 23f84a851a
commit abf0ef6513
16 changed files with 577 additions and 82 deletions
+32 -1
View File
@@ -72,7 +72,9 @@ Browser targets, pages, page elements, and dialogs are opaque capabilities. Reta
### Maintainer live-proof rig
The repository includes a macOS-only development rig that preserves the real vertical path: agent-facing `computer` tool, Gateway `node.invoke`, paired Mac node, and the selected node-local provider. It is deliberately isolated from the operator app and Gateway.
The repository includes a development rig that preserves the real vertical path: agent-facing `computer` tool, Gateway `node.invoke`, paired node, and the selected node-local provider. It is deliberately isolated from the operator app and Gateway. The macOS path uses the signed app node; the Linux path uses the opt-in `cua-computer` plugin in a real X11 session.
#### macOS
Build a signed app from a clean, committed checkout, choose a fresh profile and non-default loopback port, and prepare the two config views:
@@ -95,6 +97,35 @@ scripts/dev/computer-use-macos-live-rig.sh proof \
The proof runner first requires the sole connected computer node to advertise the requested provider, then executes `screenshot`, `list_windows`, `get_window_state`, background element click and type, and re-observes the window. It saves the structured result and target-window before/after images under the scratch directory and fails unless the provider matches, the target started non-frontmost, the frontmost app and cursor stayed unchanged, target content changed, and the final effect was confirmed or a structured refusal. Restart the isolated app with the other provider and rerun the same proof. Do not use port `18789`, the default profile, or `/Applications/OpenClaw.app` for this rig.
#### Linux X11 through Crabbox
Run Linux proof on a Crabbox Linux host, not on a macOS container. A direct AWS Crabbox lease with Xvfb is sufficient because Xvfb is a real X11 server; a local container on macOS is not remote Linux desktop proof. Install the X11 fixture prerequisites on the disposable host, then start an isolated session:
```bash
sudo apt-get update
sudo apt-get install -y at-spi2-core dbus-x11 gir1.2-gtk-3.0 jq openbox python3-gi x11-utils xdotool xvfb
dbus-run-session -- bash
export DISPLAY=:99 XDG_SESSION_TYPE=x11 NO_AT_BRIDGE=0
Xvfb "$DISPLAY" -screen 0 1280x800x24 -nolisten tcp &
openbox >/tmp/openclaw-cu-openbox.log 2>&1 &
scratch="$(mktemp -d /tmp/openclaw-cu-live.XXXXXX)"
scripts/dev/computer-use-macos-live-rig.sh prepare-linux \
cu-linux-live-proof 29431 "$scratch"
```
Run the emitted `gateway`, `node`, and `fixture` commands in separate panes that inherit the same `DISPLAY` and `DBUS_SESSION_BUS_ADDRESS`. The isolated configs share one scratch-only random Gateway token, and the gateway silently approves loopback node-device pairing. The node command surface remains an explicit approval: run the emitted `nodes` command after the node finishes reconnecting, read `.pending[0].requestId`, and pass it to `scripts/dev/computer-use-macos-live-rig.sh approve "$scratch" <request-id>`. Rerun `nodes` until exactly one connected node advertises `provider.id: "cua-computer"`.
Execute the same proof runner against the non-frontmost GTK fixture:
```bash
scripts/dev/computer-use-macos-live-rig.sh proof \
"$scratch" cua "OpenClaw CUA X11 Target" "W3-LINUX CONFIRMED"
```
The result and `window-before.png` / `window-after.png` stay under the scratch directory. A confirmed mutation must preserve the sentinel as the active X11 window and leave the pointer unchanged. An upstream `background_unavailable` or `background_occluded` result is valid refusal evidence only when it remains structured and no foreground retry is attempted. The rig rejects native Wayland even when `DISPLAY` is also present for XWayland; switch to X11 instead of claiming Wayland coverage.
### Windows and Linux (experimental, direct SDK)
The bundled `cua-computer` plugin provides an experimental fulfiller for Windows and Linux node hosts. It is disabled by default and uses the pinned CUA Driver SDK 0.19.3 contract directly:
@@ -40,7 +40,7 @@ type CuaDriverArtifactInspectionOptions = {
platform: NodeJS.Platform;
arch: string;
linuxLibc?: "gnu" | "musl";
pluginManifestPath: string;
pluginManifest: unknown;
resolvePackageJson: (packageName: string) => string | undefined;
};
@@ -96,10 +96,10 @@ function isSha256(value: unknown): value is string {
}
function loadArtifactRecord(
manifestPath: string,
manifestValue: unknown,
key: SupportedArtifactPlatform,
): { version: string; artifact: DriverArtifactRecord } | undefined {
const value = readJson(manifestPath) as CuaDriverManifest;
const value = manifestValue as CuaDriverManifest;
const version = value.dependencies?.[DRIVER_PACKAGE];
const artifact = value.cuaDriverArtifacts?.[key];
if (
@@ -139,7 +139,7 @@ export function inspectCuaDriverArtifacts(
let accepted: ReturnType<typeof loadArtifactRecord>;
try {
accepted = loadArtifactRecord(options.pluginManifestPath, selected.key);
accepted = loadArtifactRecord(options.pluginManifest, selected.key);
} catch {
accepted = undefined;
}
@@ -0,0 +1,34 @@
import { expect, it, vi } from "vitest";
const mocks = vi.hoisted(() => ({
inspect: vi.fn(() => ({ ok: true, applicable: false }) as const),
}));
vi.mock("./driver-artifact-verification.js", () => ({
inspectCuaDriverArtifacts: mocks.inspect,
readPackageIdentity: vi.fn(),
}));
import { verifyInstalledCuaDriverArtifacts } from "./driver-artifacts.js";
it("supplies the accepted artifact record without depending on the bundled module path", () => {
verifyInstalledCuaDriverArtifacts();
expect(mocks.inspect).toHaveBeenCalledWith(
expect.objectContaining({
pluginManifest: expect.objectContaining({
dependencies: expect.objectContaining({ "@trycua/cua-driver": "0.19.3" }),
cuaDriverArtifacts: expect.objectContaining({
"win32-arm64-msvc": {
files: {
"cua_driver_node_runtime.node":
"fe025669d1614b1ac9a82d1b6a331acd15b44caef81e5bda6a0b02e1d9a4b71f",
"cua_driver_sdk.dll":
"f1f25699dbdcc05169230b8286800b69a10407abb20effd5b767629fe725f21b",
},
},
}),
}),
}),
);
});
@@ -30,17 +30,16 @@ function createArtifactFixture(
const nativeContents = "accepted native artifact";
const expectedDigest =
options.expectedDigest ?? createHash("sha256").update(nativeContents).digest("hex");
const pluginManifestPath = path.join(root, "plugin-package.json");
const sdkManifestPath = path.join(root, "sdk-package.json");
const platformPackageName = `@trycua/cua-driver-${platformKey}`;
const platformDir = path.join(root, "platform");
const platformManifestPath = path.join(platformDir, "package.json");
fs.mkdirSync(platformDir);
writeJson(pluginManifestPath, {
const pluginManifest = {
dependencies: { "@trycua/cua-driver": acceptedVersion },
cuaDriverArtifacts: { [platformKey]: { files: { [nativeFile]: expectedDigest } } },
});
};
writeJson(sdkManifestPath, {
name: "@trycua/cua-driver",
version: options.sdkVersion ?? acceptedVersion,
@@ -56,7 +55,7 @@ function createArtifactFixture(
packages.set(platformPackageName, platformManifestPath);
}
return {
pluginManifestPath,
pluginManifest,
resolvePackageJson: (packageName: string) => packages.get(packageName),
};
}
@@ -1,13 +1,12 @@
import { createRequire } from "node:module";
import path from "node:path";
import { fileURLToPath } from "node:url";
import pluginManifest from "../package.json" with { type: "json" };
import {
inspectCuaDriverArtifacts,
readPackageIdentity,
type CuaDriverArtifactVerification,
} from "./driver-artifact-verification.js";
const PLUGIN_MANIFEST_PATH = fileURLToPath(new URL("../package.json", import.meta.url));
const requireFromPlugin = createRequire(import.meta.url);
function resolvePackageJson(packageName: string): string | undefined {
@@ -50,7 +49,7 @@ export function verifyInstalledCuaDriverArtifacts(): CuaDriverArtifactVerificati
platform: process.platform,
arch: process.arch,
...(process.platform === "linux" ? { linuxLibc: detectLinuxLibc() } : {}),
pluginManifestPath: PLUGIN_MANIFEST_PATH,
pluginManifest,
resolvePackageJson,
});
return installedVerification;
@@ -0,0 +1,38 @@
#!/usr/bin/env python3
import argparse
import gi
gi.require_version("Gtk", "3.0")
from gi.repository import Gtk # noqa: E402
def main() -> None:
parser = argparse.ArgumentParser(description="OpenClaw Linux X11 computer-use fixture")
parser.add_argument("--title", required=True)
parser.add_argument("--text", required=True)
args = parser.parse_args()
window = Gtk.Window(title=args.title)
window.set_default_size(520, 180)
window.connect("destroy", Gtk.main_quit)
box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=12)
box.set_border_width(24)
label = Gtk.Label(label="Background-edit target")
label.set_xalign(0)
entry = Gtk.Entry()
entry.set_name("Editor")
entry.set_text(args.text)
entry.set_activates_default(False)
box.pack_start(label, False, False, 0)
box.pack_start(entry, False, False, 0)
window.add(box)
window.show_all()
Gtk.main()
if __name__ == "__main__":
main()
+33 -13
View File
@@ -20,7 +20,7 @@ const { values } = parseArgs({
if (values.help) {
console.log(
"Usage: computer-use-macos-live-proof.ts --provider <peekaboo|cua> --window-title <title> --text <text> --artifacts <dir> [--element-label <label>]",
"Usage: computer-use-macos-live-proof.ts --provider <peekaboo|cua> --window-title <title> --text <text> --artifacts <dir> [--element-label <label>]\nRuns on macOS or an X11 Linux session; native Wayland is intentionally refused.",
);
process.exit(0);
}
@@ -187,11 +187,29 @@ async function saveImage(name: string, result: ToolResult): Promise<string> {
return output;
}
async function frontmostApp(): Promise<string> {
const script =
'tell application "System Events" to get name of first application process whose frontmost is true';
const { stdout } = await execFileAsync("/usr/bin/osascript", ["-e", script]);
return stdout.trim();
type FrontmostState = { kind: "application" | "window"; name: string };
async function frontmostState(): Promise<FrontmostState> {
if (process.platform === "darwin") {
const script =
'tell application "System Events" to get name of first application process whose frontmost is true';
const { stdout } = await execFileAsync("/usr/bin/osascript", ["-e", script]);
return { kind: "application", name: stdout.trim() };
}
if (process.platform === "linux") {
if (process.env.XDG_SESSION_TYPE === "wayland" || process.env.WAYLAND_DISPLAY) {
throw new Error("Linux live proof requires X11; native Wayland is out of scope");
}
const { stdout: windowId } = await execFileAsync("xdotool", ["getactivewindow"]);
const { stdout: title } = await execFileAsync("xdotool", ["getwindowname", windowId.trim()]);
return { kind: "window", name: title.trim() };
}
throw new Error(`frontmost-window proof is unsupported on ${process.platform}`);
}
function isTargetFrontmost(frontmost: FrontmostState, target: JsonRecord): boolean {
const targetIdentity = frontmost.kind === "application" ? target.appName : target.title;
return typeof targetIdentity === "string" && frontmost.name === targetIdentity;
}
function cursor(result: ToolResult): { x: unknown; y: unknown } {
@@ -257,10 +275,10 @@ const targetFields = {
observationId,
deliveryMode: "background",
};
const frontmostBefore = await frontmostApp();
if (frontmostBefore === target.appName) {
const frontmostBefore = await frontmostState();
if (isTargetFrontmost(frontmostBefore, target)) {
throw new Error(
`target app ${stringValue(target.appName) || "<unknown>"} is frontmost; foreground another app and retry`,
`target ${frontmostBefore.kind} ${frontmostBefore.name || "<unknown>"} is frontmost; foreground another window and retry`,
);
}
const cursorBeforeResult = await call("get_cursor_position");
@@ -282,7 +300,7 @@ if (typed.kind === "error" || wireResult(typed.result).effect !== "confirmed") {
}
const cursorAfterResult = await call("get_cursor_position");
const frontmostAfter = await frontmostApp();
const frontmostAfter = await frontmostState();
const after = await call("get_window_state", { windowRef: target.windowRef });
const afterImage = await saveImage("window-after", after);
const afterElement = selectElement(after);
@@ -291,7 +309,8 @@ const cursorAfter = cursor(cursorAfterResult);
const finalOutcome = confirmation ?? typed;
const evidence = {
route: "agent computer tool -> Gateway node.invoke -> paired Mac node -> selected provider",
route: "agent computer tool -> Gateway node.invoke -> paired node -> selected provider",
platform: process.platform,
provider: { expected: expectedProviderId, advertised: advertisedProvider },
screenshot: resultText(screenshot),
target: {
@@ -317,8 +336,9 @@ const evidence = {
},
artifacts: { beforeImage, afterImage },
assertions: {
targetWasNotFrontmost: frontmostBefore !== target.appName,
frontmostUnchanged: frontmostBefore === frontmostAfter,
targetWasNotFrontmost: !isTargetFrontmost(frontmostBefore, target),
frontmostUnchanged:
frontmostBefore.kind === frontmostAfter.kind && frontmostBefore.name === frontmostAfter.name,
cursorUnchanged: cursorBefore.x === cursorAfter.x && cursorBefore.y === cursorAfter.y,
targetContentChanged: beforeElement.value !== afterElement.value,
confirmedEffectOrStructuredRefusal: structuredOutcome(finalOutcome),
+231 -22
View File
@@ -8,14 +8,20 @@ usage() {
cat <<'EOF'
Usage:
scripts/dev/computer-use-macos-live-rig.sh prepare <profile> <port> <app> <scratch> [peekaboo|cua]
scripts/dev/computer-use-macos-live-rig.sh prepare-linux <profile> <port> <scratch>
scripts/dev/computer-use-macos-live-rig.sh gateway <scratch>
scripts/dev/computer-use-macos-live-rig.sh app <scratch> [peekaboo|cua]
scripts/dev/computer-use-macos-live-rig.sh node <scratch>
scripts/dev/computer-use-macos-live-rig.sh fixture <scratch> <target-title> <sentinel-title> <before-text>
scripts/dev/computer-use-macos-live-rig.sh nodes <scratch>
scripts/dev/computer-use-macos-live-rig.sh approve <scratch> <request-id>
scripts/dev/computer-use-macos-live-rig.sh proof <scratch> <peekaboo|cua> <window-title> <text> [element-label]
The rig is maintainer-only and loopback-only. Run gateway and app in separate
terminals, approve the dedicated CLI device after its first `nodes` request,
then run `proof`. Never use the operator profile or port 18789.
The rig is maintainer-only and loopback-only. On macOS, run gateway and app in
separate terminals. On Linux X11, run gateway, node, and fixture in separate
terminals that share DISPLAY and DBUS_SESSION_BUS_ADDRESS. Approve the isolated
node command surface after its first request, then run proof. Never use the
operator profile or port 18789. Native Wayland is intentionally unsupported.
EOF
}
@@ -33,11 +39,27 @@ validate_provider() {
require_unoccupied_port() {
local port="$1"
if /usr/sbin/lsof -nP -iTCP:"$port" -sTCP:LISTEN -t >/dev/null 2>&1; then
if ! node - "$port" >/dev/null 2>&1 <<'NODE'; then
const net = require("node:net");
const port = Number(process.argv[2]);
const server = net.createServer();
server.once("error", () => process.exit(1));
server.listen({ host: "127.0.0.1", port, exclusive: true }, () => server.close());
NODE
fail "port $port already has a listener; choose a fresh proof port"
fi
}
require_linux_x11() {
[[ "$(uname -s)" == "Linux" ]] || fail "Linux proof must run on a Linux host"
[[ "${XDG_SESSION_TYPE:-}" != "wayland" && -z "${WAYLAND_DISPLAY:-}" ]] ||
fail "native Wayland is out of scope; switch to an X11 session"
[[ -n "${DISPLAY:-}" ]] || fail "DISPLAY is required for Linux X11 proof"
command -v xdpyinfo >/dev/null || fail "xdpyinfo is required for Linux X11 proof"
command -v xdotool >/dev/null || fail "xdotool is required for Linux X11 proof"
xdpyinfo >/dev/null 2>&1 || fail "DISPLAY does not resolve to a live X11 server"
}
load_rig() {
local scratch="$1"
[[ "$scratch" = /* ]] || fail "scratch path must be absolute"
@@ -49,14 +71,26 @@ load_rig() {
done < <(node - "$rig_path" <<'NODE'
const fs = require("node:fs");
const path = process.argv[2];
const keys = ["root", "profile", "port", "app", "appState", "gatewayConfig", "agentState"];
const macKeys = ["root", "profile", "port", "app", "appState", "gatewayConfig", "agentState"];
const linuxKeys = [
"platform", "root", "profile", "port", "gatewayConfig", "gatewayState",
"agentState", "nodeConfig", "nodeState", "display",
];
try {
const value = JSON.parse(fs.readFileSync(path, "utf8"));
if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("expected object");
const actualKeys = Object.keys(value).sort();
if (actualKeys.join("\0") !== [...keys].sort().join("\0")) throw new Error("unexpected fields");
const isMac = actualKeys.join("\0") === [...macKeys].sort().join("\0");
const isLinux = actualKeys.join("\0") === [...linuxKeys].sort().join("\0");
if (!isMac && !isLinux) throw new Error("unexpected fields");
if (!Number.isInteger(value.port)) throw new Error("port must be an integer");
const fields = [value.root, value.profile, String(value.port), value.app, value.appState, value.gatewayConfig, value.agentState];
if (isLinux && value.platform !== "linux") throw new Error("invalid platform");
const fields = isMac
? ["macos", value.root, value.profile, String(value.port), value.app, value.appState,
value.gatewayConfig, value.agentState, "", "", "", value.appState]
: [value.platform, value.root, value.profile, String(value.port), "", "",
value.gatewayConfig, value.agentState, value.nodeConfig, value.nodeState,
value.display, value.gatewayState];
if (fields.some((field) => typeof field !== "string" || field.includes("\0"))) throw new Error("invalid field");
process.stdout.write(`${fields.join("\0")}\0`);
} catch (error) {
@@ -65,27 +99,48 @@ try {
}
NODE
)
[[ ${#rig_values[@]} -eq 7 ]] || fail "invalid $rig_path"
OPENCLAW_CU_RIG_ROOT="${rig_values[0]}"
OPENCLAW_CU_RIG_PROFILE="${rig_values[1]}"
OPENCLAW_CU_RIG_PORT="${rig_values[2]}"
OPENCLAW_CU_RIG_APP="${rig_values[3]}"
OPENCLAW_CU_RIG_APP_STATE="${rig_values[4]}"
OPENCLAW_CU_RIG_GATEWAY_CONFIG="${rig_values[5]}"
OPENCLAW_CU_RIG_AGENT_STATE="${rig_values[6]}"
[[ ${#rig_values[@]} -eq 12 ]] || fail "invalid $rig_path"
OPENCLAW_CU_RIG_PLATFORM="${rig_values[0]}"
OPENCLAW_CU_RIG_ROOT="${rig_values[1]}"
OPENCLAW_CU_RIG_PROFILE="${rig_values[2]}"
OPENCLAW_CU_RIG_PORT="${rig_values[3]}"
OPENCLAW_CU_RIG_APP="${rig_values[4]}"
OPENCLAW_CU_RIG_APP_STATE="${rig_values[5]}"
OPENCLAW_CU_RIG_GATEWAY_CONFIG="${rig_values[6]}"
OPENCLAW_CU_RIG_AGENT_STATE="${rig_values[7]}"
OPENCLAW_CU_RIG_NODE_CONFIG="${rig_values[8]}"
OPENCLAW_CU_RIG_NODE_STATE="${rig_values[9]}"
OPENCLAW_CU_RIG_DISPLAY="${rig_values[10]}"
OPENCLAW_CU_RIG_GATEWAY_STATE="${rig_values[11]}"
[[ "$OPENCLAW_CU_RIG_ROOT" == "$repo_root" ]] ||
fail "rig belongs to a different checkout: $OPENCLAW_CU_RIG_ROOT"
[[ "$OPENCLAW_CU_RIG_PROFILE" =~ ^[A-Za-z0-9][A-Za-z0-9_-]+$ ]] || fail "invalid rig profile"
[[ "$OPENCLAW_CU_RIG_PORT" =~ ^[0-9]+$ ]] || fail "invalid rig port"
((OPENCLAW_CU_RIG_PORT >= 1024 && OPENCLAW_CU_RIG_PORT <= 65535)) || fail "invalid rig port"
((OPENCLAW_CU_RIG_PORT != 18789)) || fail "operator port is not valid rig state"
[[ "$OPENCLAW_CU_RIG_APP" = /* ]] || fail "invalid rig app path"
[[ "$OPENCLAW_CU_RIG_APP_STATE" == "$HOME/.openclaw-$OPENCLAW_CU_RIG_PROFILE" ]] ||
fail "rig app state does not match its profile"
[[ "$OPENCLAW_CU_RIG_GATEWAY_CONFIG" == "$scratch/gateway.json" ]] ||
fail "rig gateway config is outside its scratch directory"
[[ "$OPENCLAW_CU_RIG_AGENT_STATE" == "$scratch/agent-state" ]] ||
fail "rig agent state is outside its scratch directory"
case "$OPENCLAW_CU_RIG_PLATFORM" in
macos)
[[ "$OPENCLAW_CU_RIG_APP" = /* ]] || fail "invalid rig app path"
[[ "$OPENCLAW_CU_RIG_APP_STATE" == "$HOME/.openclaw-$OPENCLAW_CU_RIG_PROFILE" ]] ||
fail "rig app state does not match its profile"
[[ "$OPENCLAW_CU_RIG_GATEWAY_STATE" == "$OPENCLAW_CU_RIG_APP_STATE" ]] ||
fail "invalid macOS gateway state"
;;
linux)
[[ "$OPENCLAW_CU_RIG_NODE_CONFIG" == "$scratch/node.json" ]] ||
fail "rig node config is outside its scratch directory"
[[ "$OPENCLAW_CU_RIG_NODE_STATE" == "$scratch/node-state" ]] ||
fail "rig node state is outside its scratch directory"
[[ "$OPENCLAW_CU_RIG_GATEWAY_STATE" == "$scratch/gateway-state" ]] ||
fail "rig gateway state is outside its scratch directory"
[[ -n "$OPENCLAW_CU_RIG_DISPLAY" ]] || fail "invalid Linux display"
;;
*) fail "invalid rig platform" ;;
esac
}
prepare() {
@@ -115,9 +170,9 @@ prepare() {
codesign --verify --deep --strict "$app_path" >/dev/null 2>&1 ||
fail "app is not a valid signed bundle: $app_path"
git -C "$repo_root" diff --quiet -- src packages extensions scripts/run-node.mjs scripts/run-node.mts ||
git -C "$repo_root" diff --quiet -- src packages extensions scripts ||
fail "runtime sources are dirty; commit and rebuild before launching the node worker"
git -C "$repo_root" diff --cached --quiet -- src packages extensions scripts/run-node.mjs scripts/run-node.mts ||
git -C "$repo_root" diff --cached --quiet -- src packages extensions scripts ||
fail "runtime sources are staged but uncommitted; commit and rebuild first"
local app_state="$HOME/.openclaw-$profile"
@@ -184,20 +239,108 @@ NODE
echo "nodes: $0 nodes $scratch"
}
prepare_linux() {
[[ $# -eq 3 ]] || { usage; exit 2; }
local profile="$1"
local port="$2"
local scratch="$3"
require_linux_x11
[[ "$profile" =~ ^[A-Za-z0-9][A-Za-z0-9_-]+$ ]] ||
fail "profile must contain only letters, digits, underscores, and dashes"
case "$profile" in
default | main | local) fail "choose a fresh, explicitly isolated profile" ;;
esac
[[ "$port" =~ ^[0-9]+$ ]] || fail "port must be numeric"
((port >= 1024 && port <= 65535)) || fail "port must be between 1024 and 65535"
((port != 18789)) || fail "port 18789 belongs to the operator gateway"
[[ "$scratch" = /* ]] || fail "scratch path must be absolute"
require_unoccupied_port "$port"
git -C "$repo_root" diff --quiet -- src packages extensions scripts ||
fail "runtime sources are dirty; commit and rebuild before launching the node worker"
git -C "$repo_root" diff --cached --quiet -- src packages extensions scripts ||
fail "runtime sources are staged but uncommitted; commit and rebuild first"
[[ ! -e "$scratch/rig.json" ]] || fail "$scratch already contains a rig"
mkdir -p "$scratch/agent-state" "$scratch/gateway-state" "$scratch/node-state"
local gateway_config="$scratch/gateway.json"
local node_config="$scratch/node.json"
local gateway_token
gateway_token="$(node -e 'process.stdout.write(require("node:crypto").randomBytes(32).toString("hex"))')"
node - "$port" "$gateway_token" >"$gateway_config" <<'NODE'
const port = Number(process.argv[2]);
const token = process.argv[3];
process.stdout.write(`${JSON.stringify({
gateway: {
mode: "local",
port,
auth: { mode: "token", token },
nodes: {
commands: { allow: ["computer.act"] },
pairing: { autoApproveLocal: true, sshVerify: false },
},
},
}, null, 2)}\n`);
NODE
node - "$port" "$gateway_token" >"$node_config" <<'NODE'
const port = Number(process.argv[2]);
const token = process.argv[3];
process.stdout.write(`${JSON.stringify({
gateway: {
mode: "remote",
remote: { transport: "direct", url: `ws://127.0.0.1:${port}`, token },
},
browser: { enabled: false },
nodeHost: { browserProxy: { enabled: false } },
plugins: { entries: { "cua-computer": { enabled: true } } },
}, null, 2)}\n`);
NODE
node - "$repo_root" "$profile" "$port" "$gateway_config" "$scratch/gateway-state" \
"$scratch/agent-state" "$node_config" "$scratch/node-state" "$DISPLAY" >"$scratch/rig.json" <<'NODE'
const [root, profile, port, gatewayConfig, gatewayState, agentState, nodeConfig, nodeState, display] = process.argv.slice(2);
process.stdout.write(`${JSON.stringify({
platform: "linux",
root,
profile,
port: Number(port),
gatewayConfig,
gatewayState,
agentState,
nodeConfig,
nodeState,
display,
}, null, 2)}\n`);
NODE
chmod 600 "$gateway_config" "$node_config" "$scratch/rig.json"
echo "prepared isolated Linux X11 profile $profile on ws://127.0.0.1:$port ($DISPLAY)"
echo "gateway: $0 gateway $scratch"
echo "node: $0 node $scratch"
echo "fixture: $0 fixture $scratch 'OpenClaw CUA X11 Target' 'OpenClaw X11 Sentinel' 'W3-LINUX BEFORE'"
echo "nodes: $0 nodes $scratch"
}
run_gateway() {
[[ $# -eq 1 ]] || { usage; exit 2; }
load_rig "$1"
require_unoccupied_port "$OPENCLAW_CU_RIG_PORT"
local auth_mode="none"
[[ "$OPENCLAW_CU_RIG_PLATFORM" == "linux" ]] && auth_mode="token"
exec env \
OPENCLAW_CONFIG_PATH="$OPENCLAW_CU_RIG_GATEWAY_CONFIG" \
OPENCLAW_STATE_DIR="$OPENCLAW_CU_RIG_APP_STATE" \
OPENCLAW_STATE_DIR="$OPENCLAW_CU_RIG_GATEWAY_STATE" \
node "$repo_root/scripts/run-node.mjs" --profile "$OPENCLAW_CU_RIG_PROFILE" \
gateway run --port "$OPENCLAW_CU_RIG_PORT" --auth none --verbose
gateway run --port "$OPENCLAW_CU_RIG_PORT" --auth "$auth_mode" --verbose
}
run_app() {
[[ $# -ge 1 && $# -le 2 ]] || { usage; exit 2; }
load_rig "$1"
[[ "$OPENCLAW_CU_RIG_PLATFORM" == "macos" ]] || fail "app is available only for macOS rigs"
local provider="${2:-peekaboo}"
validate_provider "$provider"
defaults write "ai.openclaw.mac.profile.$OPENCLAW_CU_RIG_PROFILE" \
@@ -206,6 +349,54 @@ run_app() {
"$OPENCLAW_CU_RIG_APP/Contents/MacOS/OpenClaw"
}
run_node() {
[[ $# -eq 1 ]] || { usage; exit 2; }
load_rig "$1"
[[ "$OPENCLAW_CU_RIG_PLATFORM" == "linux" ]] || fail "node is available only for Linux rigs"
require_linux_x11
[[ "$DISPLAY" == "$OPENCLAW_CU_RIG_DISPLAY" ]] || fail "DISPLAY does not match rig state"
exec env \
DISPLAY="$OPENCLAW_CU_RIG_DISPLAY" \
OPENCLAW_CONFIG_PATH="$OPENCLAW_CU_RIG_NODE_CONFIG" \
OPENCLAW_STATE_DIR="$OPENCLAW_CU_RIG_NODE_STATE" \
node "$repo_root/scripts/run-node.mjs" --profile "$OPENCLAW_CU_RIG_PROFILE" \
node run --host 127.0.0.1 --port "$OPENCLAW_CU_RIG_PORT" \
--display-name "OpenClaw CUA X11 Live Proof"
}
run_fixture() {
[[ $# -eq 4 ]] || { usage; exit 2; }
local scratch="$1"
load_rig "$scratch"
[[ "$OPENCLAW_CU_RIG_PLATFORM" == "linux" ]] || fail "fixture is available only for Linux rigs"
require_linux_x11
[[ "$DISPLAY" == "$OPENCLAW_CU_RIG_DISPLAY" ]] || fail "DISPLAY does not match rig state"
command -v python3 >/dev/null || fail "python3 is required for the Linux proof fixture"
command -v xmessage >/dev/null || fail "xmessage is required for the Linux proof sentinel"
local target_title="$2"
local sentinel_title="$3"
local before_text="$4"
local target_pid sentinel_pid target_window sentinel_window
python3 "$repo_root/scripts/dev/computer-use-linux-x11-fixture.py" \
--title "$target_title" --text "$before_text" &
target_pid=$!
xmessage -title "$sentinel_title" "This window must remain frontmost during proof." &
sentinel_pid=$!
trap 'kill "$target_pid" "$sentinel_pid" >/dev/null 2>&1 || true' EXIT INT TERM
for _ in {1..100}; do
target_window="$(xdotool search --name "$target_title" 2>/dev/null | head -n 1 || true)"
sentinel_window="$(xdotool search --name "$sentinel_title" 2>/dev/null | head -n 1 || true)"
[[ -n "$target_window" && -n "$sentinel_window" ]] && break
sleep 0.1
done
[[ -n "$target_window" && -n "$sentinel_window" ]] || fail "fixture windows did not appear"
xdotool windowactivate --sync "$sentinel_window"
echo "fixture ready: target=$target_window sentinel=$sentinel_window"
wait "$target_pid"
}
run_nodes() {
[[ $# -eq 1 ]] || { usage; exit 2; }
load_rig "$1"
@@ -215,12 +406,26 @@ run_nodes() {
node "$repo_root/scripts/run-node.mjs" nodes list --json
}
run_approve() {
[[ $# -eq 2 ]] || { usage; exit 2; }
load_rig "$1"
exec env \
OPENCLAW_CONFIG_PATH="$OPENCLAW_CU_RIG_GATEWAY_CONFIG" \
OPENCLAW_STATE_DIR="$OPENCLAW_CU_RIG_AGENT_STATE" \
node "$repo_root/scripts/run-node.mjs" nodes approve "$2" --json
}
run_proof() {
[[ $# -ge 4 && $# -le 5 ]] || { usage; exit 2; }
local scratch="$1"
load_rig "$scratch"
local provider="$2"
validate_provider "$provider"
if [[ "$OPENCLAW_CU_RIG_PLATFORM" == "linux" ]]; then
[[ "$provider" == "cua" ]] || fail "Linux rig supports only the CUA provider"
require_linux_x11
[[ "$DISPLAY" == "$OPENCLAW_CU_RIG_DISPLAY" ]] || fail "DISPLAY does not match rig state"
fi
local args=(
--provider "$provider"
--window-title "$3"
@@ -241,9 +446,13 @@ command_name="${1:-}"
shift
case "$command_name" in
prepare) prepare "$@" ;;
prepare-linux) prepare_linux "$@" ;;
gateway) run_gateway "$@" ;;
app) run_app "$@" ;;
node) run_node "$@" ;;
fixture) run_fixture "$@" ;;
nodes) run_nodes "$@" ;;
approve) run_approve "$@" ;;
proof) run_proof "$@" ;;
-h | --help | help) usage ;;
*) usage; exit 2 ;;
@@ -0,0 +1,11 @@
import type { ComputerUseCapabilityDescriptor } from "../plugins/computer-use-contract.js";
/** Publish Computer Use metadata only after the command pair is effective for this session. */
export function resolveEffectiveComputerUseDescriptor(params: {
commands: readonly string[];
declared?: ComputerUseCapabilityDescriptor;
}): ComputerUseCapabilityDescriptor | undefined {
return params.commands.includes("computer.act") && params.commands.includes("screen.snapshot")
? params.declared
: undefined;
}
+2 -4
View File
@@ -10,10 +10,8 @@ import type { ConnectParams } from "../../packages/gateway-protocol/src/index.js
import type { NodePairingRequestInput, PairedDeviceNode } from "../infra/device-pairing-node.js";
import { createEmptyPluginRegistry } from "../plugins/registry-empty.js";
import { resetPluginRuntimeStateForTest, setActivePluginRegistry } from "../plugins/runtime.js";
import {
reconcileNodePairingOnConnect,
resolveEffectiveComputerUseDescriptor,
} from "./node-connect-reconcile.js";
import { resolveEffectiveComputerUseDescriptor } from "./node-computer-use-descriptor.js";
import { reconcileNodePairingOnConnect } from "./node-connect-reconcile.js";
function makeNodeConnectParams(overrides?: Partial<ConnectParams>): ConnectParams {
return {
-10
View File
@@ -33,16 +33,6 @@ type NodeConnectPairingReconcileResult = {
shouldClearPendingPairings?: boolean;
};
/** Publish Computer Use metadata only after the command pair is effective for this session. */
export function resolveEffectiveComputerUseDescriptor(params: {
commands: readonly string[];
declared?: ComputerUseCapabilityDescriptor;
}): ComputerUseCapabilityDescriptor | undefined {
return params.commands.includes("computer.act") && params.commands.includes("screen.snapshot")
? params.declared
: undefined;
}
function resolveApprovedReconnectCommands(params: {
pairedCommands: readonly string[] | undefined;
allowlist: Set<string>;
+27 -13
View File
@@ -109,6 +109,7 @@ function makeClient(
caps?: string[];
commands?: string[];
computerUse?: unknown;
declaredComputerUse?: unknown;
workerRuns?: WorkerAdmissionHandshake;
permissions?: Record<string, boolean>;
declaredCaps?: string[];
@@ -143,6 +144,7 @@ function makeClient(
caps: opts.caps ?? [],
commands: opts.commands ?? [],
computerUse: opts.computerUse,
declaredComputerUse: opts.declaredComputerUse,
workerRuns: opts.workerRuns,
permissions: opts.permissions,
declaredCaps: opts.declaredCaps,
@@ -295,18 +297,22 @@ function authorizeSystemRun(registry: NodeRegistry, overrides: Partial<SystemRun
});
}
function computerUseDescriptor() {
return {
contractVersion: 2 as const,
provider: { id: "fixture", label: "Fixture", generation: "generation-1" },
actions: ["screenshot", "left_click"] as const,
targets: ["screen"] as const,
deliveryModes: ["foreground"] as const,
observations: ["image"] as const,
features: { recording: false, agentCursor: false, multiDisplay: false },
};
}
describe("gateway/node-registry", () => {
it("retains the validated Computer Use declaration on the live session", () => {
const registry = createNodeRegistry();
const computerUse = {
contractVersion: 2,
provider: { id: "fixture", label: "Fixture", generation: "generation-1" },
actions: ["screenshot", "left_click"],
targets: ["screen"],
deliveryModes: ["foreground"],
observations: ["image"],
features: { recording: false, agentCursor: false, multiDisplay: false },
};
const computerUse = computerUseDescriptor();
const client = makeClient("conn-computer", "node-computer", [], {
commands: ["screen.snapshot", "computer.act"],
computerUse,
@@ -2751,11 +2757,13 @@ describe("gateway/node-registry", () => {
it("refreshes effective live surface within the declared surface", () => {
const registry = createTestNodeRegistry();
const computerUse = computerUseDescriptor();
const client = makeClient("conn-1", "node-1", [], {
caps: [],
commands: [],
declaredCaps: ["talk"],
declaredCommands: ["talk.ptt.start"],
declaredCommands: ["talk.ptt.start", "computer.act", "screen.snapshot"],
declaredComputerUse: computerUse,
declaredPermissions: { microphone: true, camera: false },
});
@@ -2765,15 +2773,21 @@ describe("gateway/node-registry", () => {
const updated = registry.updateSurface("node-1", {
caps: ["talk", "screen"],
commands: ["talk.ptt.start", "system.run"],
commands: ["talk.ptt.start", "computer.act", "screen.snapshot", "system.run"],
permissions: { microphone: true, camera: true },
});
expect(updated?.caps).toEqual(["talk"]);
expect(updated?.commands).toEqual(["talk.ptt.start"]);
expect(updated?.commands).toEqual(["talk.ptt.start", "computer.act", "screen.snapshot"]);
expect(updated?.computerUse).toEqual(computerUse);
expect(updated?.permissions).toEqual({ microphone: true, camera: false });
expect(client.connect.caps).toEqual(["talk"]);
expect((client.connect as { commands?: string[] }).commands).toEqual(["talk.ptt.start"]);
expect((client.connect as { commands?: string[] }).commands).toEqual([
"talk.ptt.start",
"computer.act",
"screen.snapshot",
]);
expect(client.connect.computerUse).toEqual(computerUse);
});
it("advances the exact live session with its approved surface generation", () => {
+15
View File
@@ -22,6 +22,7 @@ import {
parseComputerUseCapabilityDescriptor,
type ComputerUseCapabilityDescriptor,
} from "../plugins/computer-use-contract.js";
import { resolveEffectiveComputerUseDescriptor } from "./node-computer-use-descriptor.js";
import {
createRegisteredNodePluginToolDescriptorMap,
normalizeNodePluginToolDescriptors,
@@ -73,6 +74,7 @@ export type NodeSession = {
declaredCommands: string[];
sessionCommandsCeiling?: string[];
commands: string[];
declaredComputerUse?: ComputerUseCapabilityDescriptor;
computerUse?: ComputerUseCapabilityDescriptor;
/** Exact node-local build admitted for worker session hosting. */
workerRuns?: WorkerAdmissionHandshake;
@@ -467,6 +469,12 @@ export class NodeRegistry {
connect.computerUse === undefined
? undefined
: parseComputerUseCapabilityDescriptor(connect.computerUse);
const declaredComputerUseValue = (connect as { declaredComputerUse?: unknown })
.declaredComputerUse;
const declaredComputerUse =
declaredComputerUseValue === undefined
? computerUse
: parseComputerUseCapabilityDescriptor(declaredComputerUseValue);
// Session ceilings preserve protocol compatibility across later pairing
// approvals while declared* retains the durable approval surface.
const sessionCapsCeiling = Array.isArray(
@@ -519,6 +527,7 @@ export class NodeRegistry {
declaredCommands,
sessionCommandsCeiling,
commands,
...(declaredComputerUse ? { declaredComputerUse } : {}),
...(computerUse ? { computerUse } : {}),
...(workerRuns ? { workerRuns } : {}),
declaredNodePluginTools,
@@ -1004,6 +1013,12 @@ export class NodeRegistry {
const nextCommands = surface.commands.filter((command) => sessionCommandsCeiling.has(command));
node.commands = nextCommands;
(node.client.connect as { commands?: string[] }).commands = nextCommands;
const nextComputerUse = resolveEffectiveComputerUseDescriptor({
commands: nextCommands,
declared: node.declaredComputerUse,
});
node.computerUse = nextComputerUse;
node.client.connect.computerUse = nextComputerUse;
this.replaceEffectiveNodePluginTools(node);
if ("caps" in surface) {
@@ -9,10 +9,8 @@ import {
import { getPairedDevice } from "../../../infra/device-pairing.js";
import { AUTH_RATE_LIMIT_SCOPE_NODE_PAIRING } from "../../auth-rate-limit.js";
import { ADMIN_SCOPE, PAIRING_SCOPE, WRITE_SCOPE } from "../../method-scopes.js";
import {
reconcileNodePairingOnConnect,
resolveEffectiveComputerUseDescriptor,
} from "../../node-connect-reconcile.js";
import { resolveEffectiveComputerUseDescriptor } from "../../node-computer-use-descriptor.js";
import { reconcileNodePairingOnConnect } from "../../node-connect-reconcile.js";
import { filterLegacyNodeProtocolFeatures } from "../../node-legacy-protocol-filter.js";
import { withSerializedRateLimitAttempt } from "../../rate-limit-attempt-serialization.js";
import type {
@@ -185,12 +183,14 @@ export async function prepareGatewayNodeConnect(
const nodeConnectParams = connectParams as ConnectParams & {
declaredCaps?: string[];
declaredCommands?: string[];
declaredComputerUse?: unknown;
declaredPermissions?: Record<string, boolean>;
sessionCapsCeiling?: string[];
sessionCommandsCeiling?: string[];
};
nodeConnectParams.declaredCaps = reconciliation.declaredCaps;
nodeConnectParams.declaredCommands = reconciliation.declaredCommands;
nodeConnectParams.declaredComputerUse = reconciliation.declaredComputerUse;
nodeConnectParams.declaredPermissions = reconciliation.declaredPermissions;
const pluginSurfaces = pluginNodeCapabilities.map((surface) => surface.surface);
if (usesLegacyNodeProtocol) {
+4 -4
View File
@@ -69,10 +69,8 @@ import {
} from "./http-common.js";
import { ADMIN_SCOPE, PAIRING_SCOPE, WRITE_SCOPE } from "./method-scopes.js";
import { isLoopbackAddress, resolveRequestClientIp } from "./net.js";
import {
reconcileNodePairingOnConnect,
resolveEffectiveComputerUseDescriptor,
} from "./node-connect-reconcile.js";
import { resolveEffectiveComputerUseDescriptor } from "./node-computer-use-descriptor.js";
import { reconcileNodePairingOnConnect } from "./node-connect-reconcile.js";
import type { NodeReapprovalCoordinator } from "./node-reapproval-coordinator.js";
import type {
NodeConnectivityResult,
@@ -872,10 +870,12 @@ export function createWatchNodeHttpRuntime(options: WatchNodeHttpRuntimeOptions)
const registeredConnect = connect as ConnectParams & {
declaredCaps?: string[];
declaredCommands?: string[];
declaredComputerUse?: unknown;
declaredPermissions?: Record<string, boolean>;
};
registeredConnect.declaredCaps = reconciliation.declaredCaps;
registeredConnect.declaredCommands = reconciliation.declaredCommands;
registeredConnect.declaredComputerUse = reconciliation.declaredComputerUse;
registeredConnect.declaredPermissions = reconciliation.declaredPermissions;
registeredConnect.caps = reconciliation.effectiveCaps;
registeredConnect.commands = reconciliation.effectiveCommands;
+137
View File
@@ -0,0 +1,137 @@
import { spawnSync } from "node:child_process";
import { chmodSync, copyFileSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { createServer } from "node:net";
import { tmpdir } from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
const rigScriptSource = path.resolve("scripts/dev/computer-use-macos-live-rig.sh");
const fixtureRoots: string[] = [];
function runGit(root: string, ...args: string[]): void {
const result = spawnSync("git", args, { cwd: root, encoding: "utf8" });
expect(result.status, result.stderr).toBe(0);
}
function writeExecutable(filePath: string, source: string): void {
writeFileSync(filePath, source);
chmodSync(filePath, 0o755);
}
function createRigRepository(): {
root: string;
script: string;
fakeBin: string;
app: string;
fixture: string;
proof: string;
} {
const root = mkdtempSync(path.join(tmpdir(), "openclaw-computer-use-rig-"));
fixtureRoots.push(root);
const scriptsDev = path.join(root, "scripts", "dev");
const fakeBin = path.join(root, "fake-bin");
const app = path.join(root, "OpenClaw.app");
const appExecutable = path.join(app, "Contents", "MacOS", "OpenClaw");
mkdirSync(scriptsDev, { recursive: true });
mkdirSync(fakeBin);
mkdirSync(path.dirname(appExecutable), { recursive: true });
const script = path.join(scriptsDev, "computer-use-macos-live-rig.sh");
const fixture = path.join(scriptsDev, "computer-use-linux-x11-fixture.py");
const proof = path.join(scriptsDev, "computer-use-macos-live-proof.ts");
copyFileSync(rigScriptSource, script);
chmodSync(script, 0o755);
writeFileSync(fixture, "# committed fixture\n");
writeFileSync(proof, "// committed proof\n");
writeExecutable(appExecutable, "#!/bin/sh\nexit 0\n");
writeExecutable(path.join(fakeBin, "codesign"), "#!/bin/sh\nexit 0\n");
writeExecutable(path.join(fakeBin, "uname"), "#!/bin/sh\necho Linux\n");
writeExecutable(path.join(fakeBin, "xdotool"), "#!/bin/sh\nexit 0\n");
writeExecutable(path.join(fakeBin, "xdpyinfo"), "#!/bin/sh\nexit 0\n");
runGit(root, "init", "-q");
runGit(root, "config", "user.name", "OpenClaw Test");
runGit(root, "config", "user.email", "openclaw-test@example.com");
runGit(root, "add", "scripts");
runGit(root, "commit", "-q", "-m", "fixture");
return { root, script, fakeBin, app, fixture, proof };
}
async function reservePort(): Promise<number> {
const server = createServer();
await new Promise<void>((resolve, reject) => {
server.once("error", reject);
server.listen({ host: "127.0.0.1", port: 0 }, resolve);
});
const address = server.address();
const port = typeof address === "object" && address ? address.port : 0;
await new Promise<void>((resolve, reject) => {
server.close((error) => {
if (error) {
reject(error);
return;
}
resolve();
});
});
return port;
}
function runRig(params: { root: string; script: string; fakeBin: string; args: string[] }) {
return spawnSync("bash", [params.script, ...params.args], {
cwd: params.root,
encoding: "utf8",
env: {
...process.env,
DISPLAY: ":99",
WAYLAND_DISPLAY: "",
XDG_SESSION_TYPE: "x11",
PATH: `${params.fakeBin}:${process.env.PATH ?? ""}`,
},
});
}
afterEach(() => {
for (const root of fixtureRoots.splice(0)) {
rmSync(root, { recursive: true, force: true });
}
});
describe("computer-use live rig source integrity", () => {
it("refuses macOS preparation after the proof runner changes", async () => {
const fixture = createRigRepository();
writeFileSync(fixture.proof, "// locally modified proof\n");
const result = runRig({
...fixture,
args: [
"prepare",
"proof-test",
String(await reservePort()),
fixture.app,
path.join(fixture.root, "scratch-mac"),
"cua",
],
});
expect(result.status).toBe(1);
expect(result.stderr).toContain("runtime sources are dirty");
});
it("refuses Linux preparation after the X11 fixture changes", async () => {
const fixture = createRigRepository();
writeFileSync(fixture.fixture, "# locally modified fixture\n");
const result = runRig({
...fixture,
args: [
"prepare-linux",
"proof-test",
String(await reservePort()),
path.join(fixture.root, "scratch-linux"),
],
});
expect(result.status).toBe(1);
expect(result.stderr).toContain("runtime sources are dirty");
});
});