mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-18 00:23:25 -06:00
refactor(computer-tool): split by responsibility (#124358)
This commit is contained in:
Peter Steinberger
committed by
GitHub
GitHub
parent
0dd8d3eafd
commit
c3488150c5
@@ -416,7 +416,6 @@ src/agents/system-prompt.ts
|
||||
src/agents/tool-display-common.ts
|
||||
src/agents/tool-loop-detection.test.ts
|
||||
src/agents/tool-search.test.ts
|
||||
src/agents/tools/computer-tool.ts
|
||||
src/agents/tools/cron-tool.test.ts
|
||||
src/agents/tools/image-generate-tool.test.ts
|
||||
src/agents/tools/image-generate-tool.ts
|
||||
|
||||
@@ -0,0 +1,497 @@
|
||||
import crypto from "node:crypto";
|
||||
import { imageMimeFromFormat } from "@openclaw/media-core/mime";
|
||||
import { isRecord } from "@openclaw/normalization-core/record-coerce";
|
||||
import { parseScreenSnapshotPayload } from "../../cli/nodes-screen.js";
|
||||
import { formatErrorMessage } from "../../infra/errors.js";
|
||||
import type {
|
||||
ComputerActParams,
|
||||
ComputerActResult,
|
||||
ComputerUseCapabilityDescriptor,
|
||||
ComputerUseV2ActionName,
|
||||
ScreenSnapshotParams,
|
||||
} from "../../plugins/computer-use-contract.js";
|
||||
import {
|
||||
COMPUTER_CONTRACT_MISMATCH,
|
||||
parseComputerActResult,
|
||||
} from "../../plugins/computer-use-contract.js";
|
||||
import { computerActionNeedsFrame, validateCapabilityBoundInput } from "./computer-tool-request.js";
|
||||
import type {
|
||||
ComputerContextEpoch,
|
||||
ComputerFrame,
|
||||
ComputerObservationState,
|
||||
ComputerTarget,
|
||||
ComputerToolAction,
|
||||
ResolvedComputerTarget,
|
||||
ScreenshotCapture,
|
||||
} from "./computer-tool-shared.js";
|
||||
import {
|
||||
COMPUTER_ACT_COMMAND,
|
||||
SCREENSHOT_QUALITY,
|
||||
SCREEN_SNAPSHOT_COMMAND,
|
||||
} from "./computer-tool-shared.js";
|
||||
import { callGatewayTool, type GatewayCallOptions } from "./gateway.js";
|
||||
import {
|
||||
type EligibleNodeMessages,
|
||||
listNodes,
|
||||
type NodeListNode,
|
||||
resolveEligibleNodeFromList,
|
||||
} from "./nodes-utils.js";
|
||||
|
||||
type ComputerState =
|
||||
| { kind: "unbound" }
|
||||
| { kind: "target"; target: ComputerTarget }
|
||||
| ({ kind: "frame" } & ComputerFrame);
|
||||
|
||||
const NOT_COMPUTER_CAPABLE_HINT =
|
||||
"enable Computer Control in the OpenClaw app and approve the pairing update";
|
||||
const DANGEROUS_DENY_HINT = "blocked by gateway.nodes.commands.deny";
|
||||
const PLATFORM_ALLOWLIST_HINT = "is not in the allowlist for platform";
|
||||
const BUTTON_NOT_HELD_HINT = "left button is not held by computer control";
|
||||
const DEFINITIVE_NODE_COMMAND_REASONS = new Set([
|
||||
"command required",
|
||||
"command not allowlisted",
|
||||
"command not declared by node",
|
||||
"node did not declare commands",
|
||||
]);
|
||||
|
||||
function isEligibleComputerNode(node: NodeListNode): boolean {
|
||||
const commands = Array.isArray(node.commands) ? node.commands : [];
|
||||
// The tool loop authorizes coordinates against captured frames, so screenshot
|
||||
// support is a functional requirement rather than gating by platform name.
|
||||
return (
|
||||
node.connected === true &&
|
||||
commands.includes(COMPUTER_ACT_COMMAND) &&
|
||||
commands.includes(SCREEN_SNAPSHOT_COMMAND)
|
||||
);
|
||||
}
|
||||
|
||||
const COMPUTER_NODE_MESSAGES: EligibleNodeMessages = {
|
||||
ineligibleExact: (query, eligibleIds) =>
|
||||
`node "${query}" is not computer-capable (needs a connected node advertising ${COMPUTER_ACT_COMMAND} and ${SCREEN_SNAPSHOT_COMMAND}; ${NOT_COMPUTER_CAPABLE_HINT}; ` +
|
||||
`eligible node ids: ${eligibleIds})`,
|
||||
nameResolveFailed: (reason, eligibleIds) =>
|
||||
`${reason} (eligible computer-capable node ids: ${eligibleIds})`,
|
||||
noneEligible: () =>
|
||||
`no connected computer-capable node (a node must advertise ${COMPUTER_ACT_COMMAND} and ${SCREEN_SNAPSHOT_COMMAND}; ${NOT_COMPUTER_CAPABLE_HINT})`,
|
||||
multipleEligible: (eligible) =>
|
||||
`multiple computer-capable nodes connected; pass node explicitly: ${eligible
|
||||
.map((node) => node.nodeId)
|
||||
.join(", ")}`,
|
||||
};
|
||||
|
||||
async function resolveComputerNode(
|
||||
gatewayOpts: GatewayCallOptions,
|
||||
query?: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<NodeListNode> {
|
||||
const nodes = await listNodes(gatewayOpts, signal);
|
||||
return resolveEligibleNodeFromList(nodes, query, isEligibleComputerNode, COMPUTER_NODE_MESSAGES);
|
||||
}
|
||||
|
||||
async function invokeNodeCommand(params: {
|
||||
gatewayOpts: GatewayCallOptions;
|
||||
nodeId: string;
|
||||
command: string;
|
||||
commandParams: Record<string, unknown>;
|
||||
timeoutMs?: number;
|
||||
idempotencyKey?: string;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<unknown> {
|
||||
const raw = await callGatewayTool<{ payload: unknown }>(
|
||||
"node.invoke",
|
||||
params.gatewayOpts,
|
||||
{
|
||||
nodeId: params.nodeId,
|
||||
command: params.command,
|
||||
params: params.commandParams,
|
||||
timeoutMs: params.timeoutMs,
|
||||
idempotencyKey: params.idempotencyKey ?? crypto.randomUUID(),
|
||||
},
|
||||
{ signal: params.signal },
|
||||
);
|
||||
return raw && typeof raw === "object" && Object.hasOwn(raw, "payload")
|
||||
? (raw as { payload: unknown }).payload
|
||||
: raw;
|
||||
}
|
||||
|
||||
function parseComputerActPayload(value: unknown): ComputerActResult {
|
||||
if (typeof value !== "string") {
|
||||
return parseComputerActResult(value);
|
||||
}
|
||||
try {
|
||||
return parseComputerActResult(JSON.parse(value));
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.message.startsWith(COMPUTER_CONTRACT_MISMATCH)) {
|
||||
throw error;
|
||||
}
|
||||
throw new Error(`${COMPUTER_CONTRACT_MISMATCH}: computer.act returned invalid JSON`, {
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function computerActIdempotencyKey(params: { scope?: string; toolCallId: string }): string {
|
||||
const stableScope = params.scope?.trim();
|
||||
const stableCallId = params.toolCallId.trim();
|
||||
if (!stableScope || !stableCallId) {
|
||||
// A call id is only unique inside its model response. Without a stable run
|
||||
// scope and provider/fallback id, avoid collapsing unrelated actions.
|
||||
return crypto.randomUUID();
|
||||
}
|
||||
const digest = crypto
|
||||
.createHash("sha256")
|
||||
.update(JSON.stringify([stableScope, stableCallId, COMPUTER_ACT_COMMAND]))
|
||||
.digest("hex");
|
||||
return `computer.act:v1:${digest}`;
|
||||
}
|
||||
|
||||
function gatewayRequestDetails(err: unknown): Record<string, unknown> | undefined {
|
||||
if (!(err instanceof Error) || err.name !== "GatewayClientRequestError") {
|
||||
return undefined;
|
||||
}
|
||||
const details = (err as Error & { details?: unknown }).details;
|
||||
return isRecord(details) ? details : undefined;
|
||||
}
|
||||
|
||||
function withComputerEnablementHint(err: unknown): Error {
|
||||
const message = formatErrorMessage(err);
|
||||
const reason = gatewayRequestDetails(err)?.reason;
|
||||
if (message.includes(DANGEROUS_DENY_HINT)) {
|
||||
return new Error(
|
||||
`${message} — remove ${COMPUTER_ACT_COMMAND} from gateway.nodes.commands.deny, then retry.`,
|
||||
{ cause: err },
|
||||
);
|
||||
}
|
||||
if (
|
||||
reason === "command not allowlisted" ||
|
||||
reason === "command not declared by node" ||
|
||||
reason === "node did not declare commands" ||
|
||||
message.includes(PLATFORM_ALLOWLIST_HINT)
|
||||
) {
|
||||
return new Error(`${message} — ${NOT_COMPUTER_CAPABLE_HINT}, then retry.`, { cause: err });
|
||||
}
|
||||
return err instanceof Error ? err : new Error(message);
|
||||
}
|
||||
|
||||
function isDefinitiveComputerActRejection(err: unknown): boolean {
|
||||
const details = gatewayRequestDetails(err);
|
||||
return (
|
||||
details?.nodeCommandDispatched === false ||
|
||||
(typeof details?.reason === "string" && DEFINITIVE_NODE_COMMAND_REASONS.has(details.reason))
|
||||
);
|
||||
}
|
||||
|
||||
function isButtonAlreadyReleasedError(err: unknown): boolean {
|
||||
return (
|
||||
err instanceof Error &&
|
||||
err.name === "GatewayClientRequestError" &&
|
||||
err.message.includes(BUTTON_NOT_HELD_HINT)
|
||||
);
|
||||
}
|
||||
|
||||
export class ComputerToolSession {
|
||||
private selectedCapabilities: ComputerUseCapabilityDescriptor | undefined;
|
||||
private selectedCapabilityNodeId: string | undefined;
|
||||
private observationState: ComputerObservationState | undefined;
|
||||
private computerState: ComputerState = { kind: "unbound" };
|
||||
private heldButtonTarget: ComputerTarget | undefined;
|
||||
private readonly executionNodes = new Map<string, GatewayCallOptions>();
|
||||
private disposePromise: Promise<void> | undefined;
|
||||
|
||||
constructor(
|
||||
private readonly options: {
|
||||
executionId: string;
|
||||
idempotencyScope?: string;
|
||||
contextEpoch?: ComputerContextEpoch;
|
||||
availableActions: (
|
||||
actions: readonly ComputerUseV2ActionName[],
|
||||
) => readonly ComputerUseV2ActionName[];
|
||||
defaultActions: readonly ComputerUseV2ActionName[];
|
||||
onCapabilitiesChanged: (capabilities?: ComputerUseCapabilityDescriptor) => void;
|
||||
registerRunCleanup?: (cleanup: (reason: string) => Promise<void>) => void;
|
||||
getOperationQueue: () => Promise<unknown>;
|
||||
},
|
||||
) {
|
||||
options.registerRunCleanup?.((reason) => this.dispose(reason));
|
||||
}
|
||||
|
||||
private bindNodeCapabilities(node: NodeListNode): void {
|
||||
const next = node.computerUse;
|
||||
const changed =
|
||||
this.selectedCapabilityNodeId !== node.nodeId ||
|
||||
this.selectedCapabilities?.provider.generation !== next?.provider.generation;
|
||||
this.selectedCapabilityNodeId = node.nodeId;
|
||||
this.selectedCapabilities = next;
|
||||
this.options.onCapabilitiesChanged(next);
|
||||
if (changed) {
|
||||
this.observationState = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
private setComputerState(next: ComputerState): void {
|
||||
this.computerState = next;
|
||||
if (!this.options.contextEpoch) {
|
||||
return;
|
||||
}
|
||||
if (next.kind !== "frame") {
|
||||
delete this.options.contextEpoch.frameToolCallId;
|
||||
delete this.options.contextEpoch.frameImageIdentity;
|
||||
}
|
||||
}
|
||||
|
||||
setTarget(target: ComputerTarget): void {
|
||||
this.setComputerState({ kind: "target", target });
|
||||
}
|
||||
|
||||
bindDeliveredFrame(params: {
|
||||
resolved: ResolvedComputerTarget;
|
||||
capture: ScreenshotCapture;
|
||||
frameId: string;
|
||||
toolCallId: string;
|
||||
imageIdentity?: string;
|
||||
modelHasVision?: boolean;
|
||||
}): void {
|
||||
if (params.modelHasVision === false || !params.imageIdentity) {
|
||||
this.setTarget(params.resolved.target);
|
||||
return;
|
||||
}
|
||||
this.computerState = {
|
||||
kind: "frame",
|
||||
target: params.resolved.target,
|
||||
id: params.frameId,
|
||||
displayFrameId: params.capture.displayFrameId,
|
||||
contextEpoch: this.options.contextEpoch?.value ?? 0,
|
||||
};
|
||||
if (this.options.contextEpoch) {
|
||||
this.options.contextEpoch.frameToolCallId = params.toolCallId;
|
||||
this.options.contextEpoch.frameImageIdentity = params.imageIdentity;
|
||||
}
|
||||
}
|
||||
|
||||
recordObservation(resolved: ResolvedComputerTarget, result: ComputerActResult): void {
|
||||
const observationId = result.observation?.observationId;
|
||||
if (observationId && resolved.capabilities) {
|
||||
this.observationState = {
|
||||
nodeId: resolved.target.nodeId,
|
||||
providerGeneration: resolved.capabilities.provider.generation,
|
||||
observationId,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async resolveTarget(params: {
|
||||
action: ComputerToolAction;
|
||||
input: Record<string, unknown>;
|
||||
gatewayOpts: GatewayCallOptions;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<ResolvedComputerTarget> {
|
||||
const explicitNode = typeof params.input.node === "string" ? params.input.node : undefined;
|
||||
const explicitScreenIndex = (() => {
|
||||
if (params.input.screenIndex === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
if (
|
||||
typeof params.input.screenIndex !== "number" ||
|
||||
!Number.isInteger(params.input.screenIndex) ||
|
||||
params.input.screenIndex < 0
|
||||
) {
|
||||
throw new Error("screenIndex must be a non-negative integer");
|
||||
}
|
||||
return params.input.screenIndex;
|
||||
})();
|
||||
const needsFrame = computerActionNeedsFrame(params.action, params.input);
|
||||
const priorTarget =
|
||||
this.computerState.kind === "unbound" ? undefined : this.computerState.target;
|
||||
const implicitTarget = this.heldButtonTarget ?? priorTarget;
|
||||
let nodeId: string;
|
||||
if (explicitNode !== undefined) {
|
||||
const node = await resolveComputerNode(params.gatewayOpts, explicitNode, params.signal);
|
||||
nodeId = node.nodeId;
|
||||
this.bindNodeCapabilities(node);
|
||||
} else if (implicitTarget) {
|
||||
nodeId = implicitTarget.nodeId;
|
||||
} else {
|
||||
const node = await resolveComputerNode(params.gatewayOpts, undefined, params.signal);
|
||||
nodeId = node.nodeId;
|
||||
this.bindNodeCapabilities(node);
|
||||
}
|
||||
const capabilities =
|
||||
this.selectedCapabilityNodeId === nodeId ? this.selectedCapabilities : undefined;
|
||||
this.executionNodes.set(nodeId, params.gatewayOpts);
|
||||
const advertisedActions = this.options.availableActions(
|
||||
capabilities?.actions ?? this.options.defaultActions,
|
||||
);
|
||||
if (!advertisedActions.includes(params.action)) {
|
||||
throw new Error(
|
||||
`${COMPUTER_CONTRACT_MISMATCH}: node ${nodeId} does not advertise action ${params.action}`,
|
||||
);
|
||||
}
|
||||
validateCapabilityBoundInput({
|
||||
action: params.action,
|
||||
input: params.input,
|
||||
nodeId,
|
||||
capabilities,
|
||||
observationState: this.observationState,
|
||||
});
|
||||
if (this.heldButtonTarget && nodeId !== this.heldButtonTarget.nodeId) {
|
||||
throw new Error(
|
||||
`computer: left button may still be held on node ${this.heldButtonTarget.nodeId}; ` +
|
||||
"release it before targeting another node",
|
||||
);
|
||||
}
|
||||
if (
|
||||
this.heldButtonTarget &&
|
||||
explicitScreenIndex !== undefined &&
|
||||
explicitScreenIndex !== this.heldButtonTarget.screenIndex
|
||||
) {
|
||||
throw new Error(
|
||||
`computer: left button may still be held on screen ${this.heldButtonTarget.screenIndex}; ` +
|
||||
"release it before targeting another screen",
|
||||
);
|
||||
}
|
||||
const targetForNode = priorTarget?.nodeId === nodeId ? priorTarget : undefined;
|
||||
const frame =
|
||||
this.computerState.kind === "frame" &&
|
||||
this.computerState.target.nodeId === nodeId &&
|
||||
this.computerState.contextEpoch === (this.options.contextEpoch?.value ?? 0)
|
||||
? this.computerState
|
||||
: undefined;
|
||||
if (needsFrame && !frame) {
|
||||
throw new Error(
|
||||
"computer: no screenshot of this node has been taken yet, so there is no display frame to " +
|
||||
"target. Take a `screenshot` first (of this node) before issuing coordinate actions.",
|
||||
);
|
||||
}
|
||||
if (
|
||||
needsFrame &&
|
||||
explicitScreenIndex !== undefined &&
|
||||
explicitScreenIndex !== frame?.target.screenIndex
|
||||
) {
|
||||
throw new Error("computer: screenIndex does not match the most recent screenshot frame");
|
||||
}
|
||||
if (needsFrame && params.input.frameId !== frame?.id) {
|
||||
throw new Error(
|
||||
"computer: frameId does not match the most recent screenshot result; take a new screenshot",
|
||||
);
|
||||
}
|
||||
const screenIndex =
|
||||
explicitScreenIndex ??
|
||||
frame?.target.screenIndex ??
|
||||
this.heldButtonTarget?.screenIndex ??
|
||||
targetForNode?.screenIndex ??
|
||||
0;
|
||||
return { target: { nodeId, screenIndex }, frame, capabilities };
|
||||
}
|
||||
|
||||
async captureScreenshot(
|
||||
resolved: ResolvedComputerTarget,
|
||||
refWidth: number,
|
||||
signal?: AbortSignal,
|
||||
): Promise<ScreenshotCapture> {
|
||||
const commandParams: ScreenSnapshotParams = {
|
||||
executionId: this.options.executionId,
|
||||
screenIndex: resolved.target.screenIndex,
|
||||
maxWidth: refWidth,
|
||||
quality: SCREENSHOT_QUALITY,
|
||||
format: "jpeg",
|
||||
};
|
||||
const payload = await invokeNodeCommand({
|
||||
gatewayOpts: this.executionNodes.get(resolved.target.nodeId)!,
|
||||
nodeId: resolved.target.nodeId,
|
||||
command: SCREEN_SNAPSHOT_COMMAND,
|
||||
commandParams,
|
||||
signal,
|
||||
});
|
||||
const parsed = parseScreenSnapshotPayload(payload);
|
||||
if (!parsed.displayFrameId) {
|
||||
throw new Error(
|
||||
"screen.snapshot response missing displayFrameId; update the node app before computer use",
|
||||
);
|
||||
}
|
||||
return {
|
||||
base64: parsed.base64,
|
||||
displayFrameId: parsed.displayFrameId,
|
||||
mimeType: imageMimeFromFormat(parsed.format) ?? "image/jpeg",
|
||||
width: parsed.width,
|
||||
height: parsed.height,
|
||||
};
|
||||
}
|
||||
|
||||
async invokeComputerAct(params: {
|
||||
resolved: ResolvedComputerTarget;
|
||||
wireParams: ComputerActParams;
|
||||
toolCallId: string;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<ComputerActResult> {
|
||||
const durationMs =
|
||||
"durationMs" in params.wireParams && typeof params.wireParams.durationMs === "number"
|
||||
? params.wireParams.durationMs
|
||||
: undefined;
|
||||
const invokeTimeoutMs = durationMs ? durationMs + 10_000 : undefined;
|
||||
params.signal?.throwIfAborted();
|
||||
this.setTarget(params.resolved.target);
|
||||
if (params.wireParams.action === "left_mouse_down") {
|
||||
this.heldButtonTarget = params.resolved.target;
|
||||
}
|
||||
let actResult: ComputerActResult;
|
||||
try {
|
||||
actResult = parseComputerActPayload(
|
||||
await invokeNodeCommand({
|
||||
gatewayOpts: this.executionNodes.get(params.resolved.target.nodeId)!,
|
||||
nodeId: params.resolved.target.nodeId,
|
||||
command: COMPUTER_ACT_COMMAND,
|
||||
commandParams: { ...params.wireParams },
|
||||
timeoutMs: invokeTimeoutMs,
|
||||
idempotencyKey: computerActIdempotencyKey({
|
||||
scope: this.options.idempotencyScope,
|
||||
toolCallId: params.toolCallId,
|
||||
}),
|
||||
signal: params.signal,
|
||||
}),
|
||||
);
|
||||
} catch (err) {
|
||||
if (params.wireParams.action === "left_mouse_down" && isDefinitiveComputerActRejection(err)) {
|
||||
this.heldButtonTarget = undefined;
|
||||
}
|
||||
if (params.wireParams.action === "left_mouse_up" && isButtonAlreadyReleasedError(err)) {
|
||||
this.heldButtonTarget = undefined;
|
||||
actResult = { ok: true };
|
||||
} else {
|
||||
throw withComputerEnablementHint(err);
|
||||
}
|
||||
}
|
||||
if (params.wireParams.action === "left_mouse_up") {
|
||||
this.heldButtonTarget = undefined;
|
||||
}
|
||||
return actResult;
|
||||
}
|
||||
|
||||
async dispose(reason: string): Promise<void> {
|
||||
if (this.disposePromise) {
|
||||
return await this.disposePromise;
|
||||
}
|
||||
this.disposePromise = this.options
|
||||
.getOperationQueue()
|
||||
.catch(() => {})
|
||||
.then(async () => {
|
||||
const nodes = [...this.executionNodes.entries()];
|
||||
this.executionNodes.clear();
|
||||
await Promise.allSettled(
|
||||
nodes.map(async ([nodeId, gatewayOpts]) => {
|
||||
await invokeNodeCommand({
|
||||
gatewayOpts,
|
||||
nodeId,
|
||||
command: COMPUTER_ACT_COMMAND,
|
||||
commandParams: {
|
||||
action: "__close_execution",
|
||||
executionId: this.options.executionId,
|
||||
reason,
|
||||
},
|
||||
idempotencyKey: `computer.close:${this.options.executionId}:${nodeId}`,
|
||||
});
|
||||
}),
|
||||
);
|
||||
});
|
||||
return await this.disposePromise;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,528 @@
|
||||
import { normalizeOptionalLowercaseString } from "@openclaw/normalization-core/string-coerce";
|
||||
import type {
|
||||
ComputerActParams,
|
||||
ComputerUseCapabilityDescriptor,
|
||||
ComputerUseV2ActionName,
|
||||
} from "../../plugins/computer-use-contract.js";
|
||||
import {
|
||||
COMPUTER_ACT_V1_ACTION_NAMES,
|
||||
COMPUTER_CONTRACT_MISMATCH,
|
||||
COMPUTER_STALE_OBSERVATION,
|
||||
COMPUTER_USE_V2_ACTION_NAMES,
|
||||
} from "../../plugins/computer-use-contract.js";
|
||||
import { readFiniteNumberParam, readPositiveIntegerParam, readToolStringParam } from "./common.js";
|
||||
import type { ComputerObservationState, ComputerToolAction } from "./computer-tool-shared.js";
|
||||
import { COMPUTER_REF_WIDTH, MAX_HOLD_SECONDS } from "./computer-tool-shared.js";
|
||||
|
||||
const LOCAL_ACTIONS = new Set<ComputerUseV2ActionName>(["screenshot", "wait"]);
|
||||
const INPUT_ACTIONS = new Set<ComputerUseV2ActionName>(
|
||||
COMPUTER_USE_V2_ACTION_NAMES.filter((action) => !LOCAL_ACTIONS.has(action)),
|
||||
);
|
||||
|
||||
const COORDINATE_REQUIRED_ACTIONS = new Set<ComputerToolAction>([
|
||||
"left_click",
|
||||
"right_click",
|
||||
"middle_click",
|
||||
"double_click",
|
||||
"triple_click",
|
||||
"mouse_move",
|
||||
"left_click_drag",
|
||||
]);
|
||||
|
||||
const ELEMENT_TARGETABLE_CLICK_ACTIONS = new Set<ComputerToolAction>([
|
||||
"left_click",
|
||||
"right_click",
|
||||
"middle_click",
|
||||
"double_click",
|
||||
"triple_click",
|
||||
]);
|
||||
|
||||
const COORDINATE_OPTIONAL_ACTIONS = new Set<ComputerToolAction>([
|
||||
"scroll",
|
||||
"left_mouse_down",
|
||||
"left_mouse_up",
|
||||
]);
|
||||
|
||||
const MODIFIER_TEXT_ACTIONS = new Set<ComputerToolAction>([
|
||||
"left_click",
|
||||
"right_click",
|
||||
"middle_click",
|
||||
"double_click",
|
||||
"triple_click",
|
||||
"left_mouse_down",
|
||||
"left_mouse_up",
|
||||
"scroll",
|
||||
]);
|
||||
|
||||
const POINTER_OR_KEYBOARD_ACTIONS = new Set<ComputerToolAction>(COMPUTER_ACT_V1_ACTION_NAMES);
|
||||
const ESCALATION_REASONS = new Set([
|
||||
"ax_tree_pixel_mismatch",
|
||||
"background_delivery_failed",
|
||||
"foreground_ineffective",
|
||||
"no_window_target",
|
||||
"other",
|
||||
]);
|
||||
const READ_ONLY_COMPUTER_ACT_ACTIONS = new Set<ComputerUseV2ActionName>([
|
||||
"list_apps",
|
||||
"list_windows",
|
||||
"get_accessibility_tree",
|
||||
"get_cursor_position",
|
||||
"get_window_state",
|
||||
"zoom",
|
||||
"get_browser_state",
|
||||
"get_recording_state",
|
||||
]);
|
||||
const SCROLL_DIRECTIONS = ["up", "down", "left", "right"] as const;
|
||||
|
||||
function isScrollDirection(value: string): value is (typeof SCROLL_DIRECTIONS)[number] {
|
||||
return SCROLL_DIRECTIONS.some((direction) => direction === value);
|
||||
}
|
||||
|
||||
export function isComputerActAction(action: ComputerToolAction): boolean {
|
||||
return INPUT_ACTIONS.has(action);
|
||||
}
|
||||
|
||||
export function isReadOnlyComputerActAction(action: ComputerToolAction): boolean {
|
||||
return READ_ONLY_COMPUTER_ACT_ACTIONS.has(action);
|
||||
}
|
||||
|
||||
export function computerActionNeedsFrame(
|
||||
action: ComputerToolAction,
|
||||
input: Record<string, unknown>,
|
||||
): boolean {
|
||||
return (
|
||||
!input.windowRef &&
|
||||
!input.elementRef &&
|
||||
(COORDINATE_REQUIRED_ACTIONS.has(action) ||
|
||||
(COORDINATE_OPTIONAL_ACTIONS.has(action) && Array.isArray(input.coordinate)))
|
||||
);
|
||||
}
|
||||
|
||||
function readCoordinate(
|
||||
params: Record<string, unknown>,
|
||||
key: "coordinate" | "startCoordinate",
|
||||
): [number, number] | undefined {
|
||||
const raw = params[key];
|
||||
if (raw === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
if (
|
||||
!Array.isArray(raw) ||
|
||||
raw.length !== 2 ||
|
||||
raw.some(
|
||||
(entry) =>
|
||||
typeof entry !== "number" ||
|
||||
!Number.isFinite(entry) ||
|
||||
!Number.isInteger(entry) ||
|
||||
entry < 0,
|
||||
)
|
||||
) {
|
||||
throw new Error(`${key} must be a pair of non-negative integers`);
|
||||
}
|
||||
return [raw[0] as number, raw[1] as number];
|
||||
}
|
||||
|
||||
function requireCoordinate(params: Record<string, unknown>, action: string): [number, number] {
|
||||
const coordinate = readCoordinate(params, "coordinate");
|
||||
if (!coordinate) {
|
||||
throw new Error(`coordinate [x, y] required for ${action}`);
|
||||
}
|
||||
return [coordinate[0], coordinate[1]];
|
||||
}
|
||||
|
||||
function readModifiers(params: Record<string, unknown>, action: ComputerToolAction) {
|
||||
if (!MODIFIER_TEXT_ACTIONS.has(action)) {
|
||||
return undefined;
|
||||
}
|
||||
const text = typeof params.text === "string" ? params.text.trim() : "";
|
||||
return text ? text : undefined;
|
||||
}
|
||||
|
||||
function copyOptionalStringParam(
|
||||
target: Record<string, unknown>,
|
||||
input: Record<string, unknown>,
|
||||
key: string,
|
||||
): void {
|
||||
const value = readToolStringParam(input, key);
|
||||
if (value !== undefined) {
|
||||
target[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
function copyOptionalIntegerParam(
|
||||
target: Record<string, unknown>,
|
||||
input: Record<string, unknown>,
|
||||
key: string,
|
||||
bounds: { min: number; max: number },
|
||||
): void {
|
||||
const value = readFiniteNumberParam(input, key, bounds);
|
||||
if (value === undefined) {
|
||||
return;
|
||||
}
|
||||
if (!Number.isInteger(value)) {
|
||||
throw new Error(`${key} must be an integer`);
|
||||
}
|
||||
target[key] = value;
|
||||
}
|
||||
|
||||
function copyDeliveryMode(target: Record<string, unknown>, input: Record<string, unknown>): void {
|
||||
const deliveryMode = normalizeOptionalLowercaseString(input.deliveryMode);
|
||||
if (deliveryMode === undefined) {
|
||||
return;
|
||||
}
|
||||
if (deliveryMode !== "background" && deliveryMode !== "foreground") {
|
||||
throw new Error("deliveryMode must be background or foreground");
|
||||
}
|
||||
target.deliveryMode = deliveryMode;
|
||||
}
|
||||
|
||||
function copyOptionalBooleanParam(
|
||||
target: Record<string, unknown>,
|
||||
input: Record<string, unknown>,
|
||||
key: string,
|
||||
): void {
|
||||
const value = input[key];
|
||||
if (value === undefined) {
|
||||
return;
|
||||
}
|
||||
if (typeof value !== "boolean") {
|
||||
throw new Error(`${key} must be a boolean`);
|
||||
}
|
||||
target[key] = value;
|
||||
}
|
||||
|
||||
function copyBrowserRefs(target: Record<string, unknown>, input: Record<string, unknown>): void {
|
||||
target.browserRef = readToolStringParam(input, "browserRef", { required: true });
|
||||
target.pageRef = readToolStringParam(input, "pageRef", { required: true });
|
||||
}
|
||||
|
||||
/** Builds the computer.act wire params for one tool input action. */
|
||||
export function buildComputerActParams(params: {
|
||||
action: ComputerToolAction;
|
||||
input: Record<string, unknown>;
|
||||
executionId: string;
|
||||
screenIndex: number;
|
||||
displayFrameId?: string;
|
||||
refWidth?: number;
|
||||
}): ComputerActParams {
|
||||
const { action, input } = params;
|
||||
const wire: Record<string, unknown> = { action, executionId: params.executionId };
|
||||
if ((COMPUTER_ACT_V1_ACTION_NAMES as readonly string[]).includes(action)) {
|
||||
wire.screenIndex = params.screenIndex;
|
||||
wire.refWidth = params.refWidth ?? COMPUTER_REF_WIDTH;
|
||||
}
|
||||
const elementRef = readToolStringParam(input, "elementRef");
|
||||
if (
|
||||
COORDINATE_REQUIRED_ACTIONS.has(action) &&
|
||||
!(elementRef && ELEMENT_TARGETABLE_CLICK_ACTIONS.has(action))
|
||||
) {
|
||||
const [x, y] = requireCoordinate(input, action);
|
||||
wire.x = x;
|
||||
wire.y = y;
|
||||
} else if (COORDINATE_OPTIONAL_ACTIONS.has(action)) {
|
||||
const coordinate = readCoordinate(input, "coordinate");
|
||||
if (coordinate) {
|
||||
wire.x = coordinate[0];
|
||||
wire.y = coordinate[1];
|
||||
}
|
||||
}
|
||||
if ((wire.x !== undefined || wire.fromX !== undefined) && params.displayFrameId) {
|
||||
wire.displayFrameId = params.displayFrameId;
|
||||
}
|
||||
const modifiers = readModifiers(input, action);
|
||||
if (modifiers) {
|
||||
wire.modifiers = modifiers;
|
||||
}
|
||||
switch (action) {
|
||||
case "left_click_drag": {
|
||||
const start = readCoordinate(input, "startCoordinate");
|
||||
if (!start) {
|
||||
throw new Error("startCoordinate [x, y] required for left_click_drag");
|
||||
}
|
||||
wire.fromX = start[0];
|
||||
wire.fromY = start[1];
|
||||
break;
|
||||
}
|
||||
case "scroll": {
|
||||
const direction = normalizeOptionalLowercaseString(input.scrollDirection);
|
||||
if (!direction || !isScrollDirection(direction)) {
|
||||
throw new Error("scrollDirection up|down|left|right required for scroll");
|
||||
}
|
||||
wire.scrollDirection = direction;
|
||||
const amount = readPositiveIntegerParam(input, "scrollAmount") ?? 3;
|
||||
wire.scrollAmount = Math.min(100, amount);
|
||||
break;
|
||||
}
|
||||
case "type": {
|
||||
const text = typeof input.text === "string" ? input.text : "";
|
||||
if (!text) {
|
||||
throw new Error("text required for type");
|
||||
}
|
||||
wire.text = text;
|
||||
break;
|
||||
}
|
||||
case "key":
|
||||
case "hold_key": {
|
||||
const keys = readToolStringParam(input, "text", { required: true });
|
||||
wire.keys = keys;
|
||||
if (action === "hold_key") {
|
||||
const seconds =
|
||||
readFiniteNumberParam(input, "duration", {
|
||||
min: 0,
|
||||
minExclusive: true,
|
||||
max: MAX_HOLD_SECONDS,
|
||||
message: `duration must be >0 and <=${MAX_HOLD_SECONDS} seconds for hold_key`,
|
||||
}) ?? 1;
|
||||
wire.durationMs = Math.round(seconds * 1000);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case "get_accessibility_tree": {
|
||||
copyOptionalStringParam(wire, input, "windowRef");
|
||||
copyOptionalStringParam(wire, input, "query");
|
||||
copyOptionalIntegerParam(wire, input, "depth", { min: 0, max: 64 });
|
||||
copyOptionalIntegerParam(wire, input, "maxElements", { min: 1, max: 2_000 });
|
||||
break;
|
||||
}
|
||||
case "get_window_state": {
|
||||
wire.windowRef = readToolStringParam(input, "windowRef", { required: true });
|
||||
copyOptionalStringParam(wire, input, "query");
|
||||
copyOptionalIntegerParam(wire, input, "depth", { min: 0, max: 64 });
|
||||
copyOptionalIntegerParam(wire, input, "maxElements", { min: 1, max: 2_000 });
|
||||
break;
|
||||
}
|
||||
case "launch_app":
|
||||
case "kill_app": {
|
||||
wire.app = readToolStringParam(input, "app", { required: true });
|
||||
break;
|
||||
}
|
||||
case "bring_to_front": {
|
||||
wire.windowRef = readToolStringParam(input, "windowRef", { required: true });
|
||||
break;
|
||||
}
|
||||
case "set_value": {
|
||||
for (const key of ["windowRef", "elementRef", "observationId", "value"] as const) {
|
||||
wire[key] = readToolStringParam(input, key, {
|
||||
required: true,
|
||||
allowEmpty: key === "value",
|
||||
});
|
||||
}
|
||||
copyDeliveryMode(wire, input);
|
||||
break;
|
||||
}
|
||||
case "invoke_menu": {
|
||||
wire.windowRef = readToolStringParam(input, "windowRef", { required: true });
|
||||
const path = input.path;
|
||||
if (
|
||||
!Array.isArray(path) ||
|
||||
path.length < 1 ||
|
||||
path.length > 16 ||
|
||||
path.some((segment) => typeof segment !== "string" || !segment.trim())
|
||||
) {
|
||||
throw new Error("path must contain 1-16 non-empty menu labels");
|
||||
}
|
||||
wire.path = path;
|
||||
copyDeliveryMode(wire, input);
|
||||
break;
|
||||
}
|
||||
case "zoom": {
|
||||
wire.windowRef = readToolStringParam(input, "windowRef", { required: true });
|
||||
wire.observationId = readToolStringParam(input, "observationId", { required: true });
|
||||
for (const key of ["x1", "y1", "x2", "y2"] as const) {
|
||||
const value = readFiniteNumberParam(input, key, { min: 0 });
|
||||
if (value === undefined) {
|
||||
throw new Error(`${key} required for zoom`);
|
||||
}
|
||||
wire[key] = value;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case "get_browser_state": {
|
||||
const windowRef = readToolStringParam(input, "windowRef");
|
||||
if (windowRef) {
|
||||
wire.windowRef = windowRef;
|
||||
break;
|
||||
}
|
||||
copyBrowserRefs(wire, input);
|
||||
for (const key of [
|
||||
"snapshotFormat",
|
||||
"elementRef",
|
||||
"observationId",
|
||||
"query",
|
||||
"continuation",
|
||||
] as const) {
|
||||
copyOptionalStringParam(wire, input, key);
|
||||
}
|
||||
copyOptionalBooleanParam(wire, input, "includeScreenshot");
|
||||
break;
|
||||
}
|
||||
case "browser_prepare": {
|
||||
wire.windowRef = readToolStringParam(input, "windowRef", { required: true });
|
||||
copyOptionalStringParam(wire, input, "profile");
|
||||
copyOptionalStringParam(wire, input, "profileName");
|
||||
break;
|
||||
}
|
||||
case "browser_navigate": {
|
||||
copyBrowserRefs(wire, input);
|
||||
wire.url = readToolStringParam(input, "url", { required: true });
|
||||
break;
|
||||
}
|
||||
case "browser_click": {
|
||||
copyBrowserRefs(wire, input);
|
||||
wire.observationId = readToolStringParam(input, "observationId", { required: true });
|
||||
copyOptionalStringParam(wire, input, "elementRef");
|
||||
copyOptionalStringParam(wire, input, "inputRoute");
|
||||
const coordinate = readCoordinate(input, "coordinate");
|
||||
if (coordinate) {
|
||||
wire.x = coordinate[0];
|
||||
wire.y = coordinate[1];
|
||||
}
|
||||
break;
|
||||
}
|
||||
case "browser_type": {
|
||||
copyBrowserRefs(wire, input);
|
||||
for (const key of ["observationId", "elementRef"] as const) {
|
||||
wire[key] = readToolStringParam(input, key, { required: true });
|
||||
}
|
||||
wire.text = readToolStringParam(input, "text", { required: true, allowEmpty: true });
|
||||
copyOptionalStringParam(wire, input, "mode");
|
||||
copyOptionalBooleanParam(wire, input, "replace");
|
||||
break;
|
||||
}
|
||||
case "browser_dialog": {
|
||||
copyBrowserRefs(wire, input);
|
||||
wire.dialogAction = readToolStringParam(input, "dialogAction", { required: true });
|
||||
copyOptionalStringParam(wire, input, "dialogRef");
|
||||
copyOptionalStringParam(wire, input, "promptText");
|
||||
copyDeliveryMode(wire, input);
|
||||
break;
|
||||
}
|
||||
case "browser_set_input_files": {
|
||||
copyBrowserRefs(wire, input);
|
||||
for (const key of ["observationId", "elementRef"] as const) {
|
||||
wire[key] = readToolStringParam(input, key, { required: true });
|
||||
}
|
||||
const resourceHandles = input.resourceHandles;
|
||||
if (
|
||||
!Array.isArray(resourceHandles) ||
|
||||
resourceHandles.length < 1 ||
|
||||
resourceHandles.length > 32 ||
|
||||
resourceHandles.some((handle) => typeof handle !== "string" || !handle)
|
||||
) {
|
||||
throw new Error("resourceHandles must contain 1-32 opaque resource handles");
|
||||
}
|
||||
wire.resourceHandles = resourceHandles;
|
||||
break;
|
||||
}
|
||||
case "browser_download": {
|
||||
copyBrowserRefs(wire, input);
|
||||
for (const key of ["observationId", "elementRef"] as const) {
|
||||
wire[key] = readToolStringParam(input, key, { required: true });
|
||||
}
|
||||
break;
|
||||
}
|
||||
case "browser_pointer": {
|
||||
copyBrowserRefs(wire, input);
|
||||
wire.observationId = readToolStringParam(input, "observationId", { required: true });
|
||||
wire.pointerAction = readToolStringParam(input, "pointerAction", { required: true });
|
||||
for (const key of ["inputRoute", "elementRef", "destinationElementRef"] as const) {
|
||||
copyOptionalStringParam(wire, input, key);
|
||||
}
|
||||
const coordinate = readCoordinate(input, "coordinate");
|
||||
if (coordinate) {
|
||||
wire.x = coordinate[0];
|
||||
wire.y = coordinate[1];
|
||||
}
|
||||
const destination = input.destinationCoordinate;
|
||||
if (destination !== undefined) {
|
||||
if (
|
||||
!Array.isArray(destination) ||
|
||||
destination.length !== 2 ||
|
||||
destination.some((value) => typeof value !== "number" || !Number.isFinite(value))
|
||||
) {
|
||||
throw new Error("destinationCoordinate must be a pair of finite numbers");
|
||||
}
|
||||
wire.toX = destination[0];
|
||||
wire.toY = destination[1];
|
||||
}
|
||||
for (const key of ["deltaX", "deltaY"] as const) {
|
||||
const value = readFiniteNumberParam(input, key);
|
||||
if (value !== undefined) {
|
||||
wire[key] = value;
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
case "escalate_scope": {
|
||||
const reason = readToolStringParam(input, "reason", { required: true });
|
||||
if (!ESCALATION_REASONS.has(reason)) {
|
||||
throw new Error("reason must be a supported escalation reason");
|
||||
}
|
||||
wire.reason = reason;
|
||||
break;
|
||||
}
|
||||
case "start_recording": {
|
||||
copyOptionalBooleanParam(wire, input, "recordVideo");
|
||||
break;
|
||||
}
|
||||
case "replay_trajectory": {
|
||||
wire.resourceHandle = readToolStringParam(input, "resourceHandle", { required: true });
|
||||
copyOptionalIntegerParam(wire, input, "delayMs", { min: 0, max: 10_000 });
|
||||
copyOptionalBooleanParam(wire, input, "stopOnError");
|
||||
break;
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
if (POINTER_OR_KEYBOARD_ACTIONS.has(action)) {
|
||||
for (const key of ["windowRef", "elementRef", "observationId"] as const) {
|
||||
copyOptionalStringParam(wire, input, key);
|
||||
}
|
||||
copyDeliveryMode(wire, input);
|
||||
}
|
||||
return wire as ComputerActParams;
|
||||
}
|
||||
|
||||
export function validateCapabilityBoundInput(params: {
|
||||
action: ComputerUseV2ActionName;
|
||||
input: Record<string, unknown>;
|
||||
nodeId: string;
|
||||
capabilities?: ComputerUseCapabilityDescriptor;
|
||||
observationState?: ComputerObservationState;
|
||||
}): void {
|
||||
const { capabilities, input } = params;
|
||||
const windowRef = readToolStringParam(input, "windowRef");
|
||||
const browserRef = readToolStringParam(input, "browserRef");
|
||||
const pageRef = readToolStringParam(input, "pageRef");
|
||||
const elementRef = readToolStringParam(input, "elementRef");
|
||||
const observationId = readToolStringParam(input, "observationId");
|
||||
const deliveryMode = normalizeOptionalLowercaseString(input.deliveryMode);
|
||||
if (windowRef && !capabilities?.targets.includes("window")) {
|
||||
throw new Error(`${COMPUTER_CONTRACT_MISMATCH}: selected node has no window target support`);
|
||||
}
|
||||
if (elementRef && !capabilities?.targets.includes("element")) {
|
||||
throw new Error(`${COMPUTER_CONTRACT_MISMATCH}: selected node has no element target support`);
|
||||
}
|
||||
if ((browserRef || pageRef) && !capabilities?.targets.includes("browser")) {
|
||||
throw new Error(`${COMPUTER_CONTRACT_MISMATCH}: selected node has no browser target support`);
|
||||
}
|
||||
if (deliveryMode && !capabilities?.deliveryModes.includes(deliveryMode as never)) {
|
||||
throw new Error(
|
||||
`${COMPUTER_CONTRACT_MISMATCH}: selected node does not advertise ${deliveryMode} delivery`,
|
||||
);
|
||||
}
|
||||
if (elementRef && !observationId) {
|
||||
throw new Error(`${COMPUTER_STALE_OBSERVATION}: elementRef requires observationId`);
|
||||
}
|
||||
if (!observationId) {
|
||||
return;
|
||||
}
|
||||
if (
|
||||
!params.observationState ||
|
||||
params.observationState.nodeId !== params.nodeId ||
|
||||
params.observationState.providerGeneration !== capabilities?.provider.generation ||
|
||||
params.observationState.observationId !== observationId
|
||||
) {
|
||||
throw new Error(`${COMPUTER_STALE_OBSERVATION}: take a fresh observation and retry`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,232 @@
|
||||
import crypto from "node:crypto";
|
||||
import { imageMimeFromFormat } from "@openclaw/media-core/mime";
|
||||
import type { ComputerActResult } from "../../plugins/computer-use-contract.js";
|
||||
import { DEFAULT_IMAGE_MAX_DIMENSION_PX } from "../image-sanitization.js";
|
||||
import type { AgentMessage, AgentToolResult } from "../runtime/index.js";
|
||||
import { sanitizeToolResultImages } from "../tool-images.js";
|
||||
import type {
|
||||
ComputerContextEpoch,
|
||||
ComputerTarget,
|
||||
ComputerToolAction,
|
||||
ScreenshotCapture,
|
||||
} from "./computer-tool-shared.js";
|
||||
import { COMPUTER_REF_WIDTH, MODEL_OBSERVATION_MAX_ELEMENTS } from "./computer-tool-shared.js";
|
||||
|
||||
type ModelObservationProjection = NonNullable<ComputerActResult["observation"]> & {
|
||||
truncatedElements?: number;
|
||||
};
|
||||
|
||||
export function computerActResultText(
|
||||
action: ComputerToolAction,
|
||||
result: ComputerActResult,
|
||||
): string {
|
||||
let observation: ModelObservationProjection | undefined = result.observation
|
||||
? { ...result.observation, ...(result.observation.base64 ? { base64: "[image]" } : {}) }
|
||||
: undefined;
|
||||
if (observation?.elements && observation.elements.length > MODEL_OBSERVATION_MAX_ELEMENTS) {
|
||||
observation = {
|
||||
...observation,
|
||||
elements: observation.elements.slice(0, MODEL_OBSERVATION_MAX_ELEMENTS),
|
||||
truncatedElements: observation.elements.length - MODEL_OBSERVATION_MAX_ELEMENTS,
|
||||
};
|
||||
}
|
||||
const details = result.details ? { ...result.details } : undefined;
|
||||
if (
|
||||
details &&
|
||||
Array.isArray(details.elements) &&
|
||||
details.elements.length > MODEL_OBSERVATION_MAX_ELEMENTS
|
||||
) {
|
||||
const originalLength = details.elements.length;
|
||||
details.elements = details.elements.slice(0, MODEL_OBSERVATION_MAX_ELEMENTS);
|
||||
details.truncatedElements = originalLength - MODEL_OBSERVATION_MAX_ELEMENTS;
|
||||
}
|
||||
return JSON.stringify({
|
||||
action,
|
||||
...result,
|
||||
...(observation ? { observation } : {}),
|
||||
...(details ? { details } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
function computerFrameImageIdentity(
|
||||
content: AgentToolResult<unknown>["content"],
|
||||
): string | undefined {
|
||||
const images = content.filter(
|
||||
(block): block is Extract<(typeof content)[number], { type: "image" }> =>
|
||||
block.type === "image",
|
||||
);
|
||||
if (images.length !== 1) {
|
||||
return undefined;
|
||||
}
|
||||
const image = images.at(0);
|
||||
if (!image) {
|
||||
return undefined;
|
||||
}
|
||||
return crypto
|
||||
.createHash("sha256")
|
||||
.update(JSON.stringify([image.mimeType, image.data]))
|
||||
.digest("hex");
|
||||
}
|
||||
|
||||
function invalidateComputerFrame(contextEpoch: ComputerContextEpoch): boolean {
|
||||
if (contextEpoch.frameToolCallId === undefined && contextEpoch.frameImageIdentity === undefined) {
|
||||
return false;
|
||||
}
|
||||
contextEpoch.value += 1;
|
||||
delete contextEpoch.frameToolCallId;
|
||||
delete contextEpoch.frameImageIdentity;
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Invalidate screenshot coordinates when the final model context no longer
|
||||
* contains the image produced by the tracked computer tool result.
|
||||
*/
|
||||
export function invalidateComputerFrameIfMissing(params: {
|
||||
contextEpoch: ComputerContextEpoch;
|
||||
messages: AgentMessage[];
|
||||
imagesBlocked?: boolean;
|
||||
}): boolean {
|
||||
const frameToolCallId = params.contextEpoch.frameToolCallId;
|
||||
if (frameToolCallId === undefined) {
|
||||
return invalidateComputerFrame(params.contextEpoch);
|
||||
}
|
||||
|
||||
let frameImageIdentity: string | undefined;
|
||||
for (let index = params.messages.length - 1; index >= 0; index -= 1) {
|
||||
const message = params.messages[index];
|
||||
if (
|
||||
message?.role !== "toolResult" ||
|
||||
message.toolName !== "computer" ||
|
||||
message.toolCallId !== frameToolCallId
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
frameImageIdentity = computerFrameImageIdentity(message.content);
|
||||
break;
|
||||
}
|
||||
|
||||
if (
|
||||
!params.imagesBlocked &&
|
||||
frameImageIdentity !== undefined &&
|
||||
frameImageIdentity === params.contextEpoch.frameImageIdentity
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
return invalidateComputerFrame(params.contextEpoch);
|
||||
}
|
||||
|
||||
/**
|
||||
* The reference frame width both the screenshot and the coordinates use.
|
||||
* Capped at the model's image sanitization limit so a persisted screenshot that
|
||||
* is replay-sanitized in a later turn is not resized underneath the coordinate
|
||||
* frame the model is still issuing `refWidth` against.
|
||||
*/
|
||||
export function resolveReferenceWidth(limits: { maxDimensionPx?: number }): number {
|
||||
const sanitizationLimit = limits.maxDimensionPx ?? DEFAULT_IMAGE_MAX_DIMENSION_PX;
|
||||
return Math.max(1, Math.min(COMPUTER_REF_WIDTH, sanitizationLimit));
|
||||
}
|
||||
|
||||
export async function projectScreenshotResult(params: {
|
||||
capture: ScreenshotCapture;
|
||||
noteLines: string[];
|
||||
target: ComputerTarget;
|
||||
action: ComputerToolAction;
|
||||
referenceWidth: number;
|
||||
modelHasVision?: boolean;
|
||||
}): Promise<{
|
||||
result: AgentToolResult<unknown>;
|
||||
frameId: string;
|
||||
imageIdentity?: string;
|
||||
}> {
|
||||
const { capture, target } = params;
|
||||
const frameId = crypto.randomUUID();
|
||||
// Report the delivered dimensions, not the pre-sanitization capture size:
|
||||
// sanitizeToolResultImages caps the longest edge to referenceWidth, so a
|
||||
// portrait capture is scaled down. Advertising the original size would let
|
||||
// the model pick coordinates against a wider frame than it was shown.
|
||||
const longestEdge = Math.max(capture.width ?? 0, capture.height ?? 0);
|
||||
const frameScale = longestEdge > params.referenceWidth ? params.referenceWidth / longestEdge : 1;
|
||||
const deliveredWidth = capture.width != null ? Math.round(capture.width * frameScale) : undefined;
|
||||
const deliveredHeight =
|
||||
capture.height != null ? Math.round(capture.height * frameScale) : undefined;
|
||||
const dims =
|
||||
deliveredWidth && deliveredHeight ? `${deliveredWidth}x${deliveredHeight}` : "unknown size";
|
||||
const text = [
|
||||
...params.noteLines,
|
||||
`screenshot ${dims} (screen ${target.screenIndex}, frameId ${frameId})`,
|
||||
].join("\n");
|
||||
const content: AgentToolResult<unknown>["content"] = [{ type: "text", text }];
|
||||
if (params.modelHasVision !== false) {
|
||||
content.push({ type: "image", data: capture.base64, mimeType: capture.mimeType });
|
||||
} else {
|
||||
content.push({
|
||||
type: "text",
|
||||
text: "[model has no vision; screenshot omitted — use a vision-capable model for computer use]",
|
||||
});
|
||||
}
|
||||
// Cap the delivered screenshot's longest edge to the reference width so
|
||||
// the coordinate frame is stable across turns. Replay-sanitization in
|
||||
// later turns caps the longest edge to the configured limit, which is
|
||||
// >= referenceWidth, so it is a no-op and the node maps coordinates
|
||||
// against this same width for both portrait and landscape captures. A
|
||||
// portrait frame (height > referenceWidth) is uniformly scaled down here,
|
||||
// matching OpenClawComputerInputGeometry.capturedWidth on the node.
|
||||
// media.outbound=false keeps desktop pixels model-only (#44759).
|
||||
const result = await sanitizeToolResultImages(
|
||||
{
|
||||
content,
|
||||
details: {
|
||||
node: target.nodeId,
|
||||
action: params.action,
|
||||
width: deliveredWidth,
|
||||
height: deliveredHeight,
|
||||
screenIndex: target.screenIndex,
|
||||
frameId,
|
||||
refWidth: params.referenceWidth,
|
||||
media: { outbound: false },
|
||||
},
|
||||
},
|
||||
`computer:${params.action}`,
|
||||
{ maxDimensionPx: params.referenceWidth },
|
||||
);
|
||||
return {
|
||||
result,
|
||||
frameId,
|
||||
imageIdentity: computerFrameImageIdentity(result.content),
|
||||
};
|
||||
}
|
||||
|
||||
export async function projectComputerActResult(params: {
|
||||
result: ComputerActResult;
|
||||
target: ComputerTarget;
|
||||
action: ComputerToolAction;
|
||||
referenceWidth: number;
|
||||
modelHasVision?: boolean;
|
||||
}): Promise<AgentToolResult<unknown>> {
|
||||
const observation = params.result.observation;
|
||||
const content: AgentToolResult<unknown>["content"] = [
|
||||
{ type: "text", text: computerActResultText(params.action, params.result) },
|
||||
];
|
||||
if (observation?.base64 && params.modelHasVision !== false) {
|
||||
content.push({
|
||||
type: "image",
|
||||
data: observation.base64,
|
||||
mimeType: imageMimeFromFormat(observation.format ?? "png") ?? "image/png",
|
||||
});
|
||||
}
|
||||
return await sanitizeToolResultImages(
|
||||
{
|
||||
content,
|
||||
details: {
|
||||
node: params.target.nodeId,
|
||||
action: params.action,
|
||||
screenIndex: params.target.screenIndex,
|
||||
result: params.result,
|
||||
media: { outbound: false },
|
||||
},
|
||||
},
|
||||
`computer:${params.action}`,
|
||||
{ maxDimensionPx: params.referenceWidth },
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
import { Type } from "typebox";
|
||||
import {
|
||||
COMPUTER_USE_V1_ACTION_NAMES,
|
||||
type ComputerUseV2ActionName,
|
||||
} from "../../plugins/computer-use-contract.js";
|
||||
import {
|
||||
optionalFiniteNumberSchema,
|
||||
optionalNonNegativeIntegerSchema,
|
||||
optionalPositiveIntegerSchema,
|
||||
optionalStringEnum,
|
||||
stringEnum,
|
||||
} from "../schema/typebox.js";
|
||||
import { MAX_HOLD_SECONDS, MAX_WAIT_SECONDS } from "./computer-tool-shared.js";
|
||||
import { gatewayCallOptionSchemaProperties } from "./gateway-schema.js";
|
||||
|
||||
export const COMPUTER_TOOL_ACTIONS = COMPUTER_USE_V1_ACTION_NAMES;
|
||||
|
||||
const EXECUTION_OWNED_ACTIONS = new Set<ComputerUseV2ActionName>([
|
||||
"browser_set_input_files",
|
||||
"browser_download",
|
||||
"get_recording_state",
|
||||
"start_recording",
|
||||
"stop_recording",
|
||||
"replay_trajectory",
|
||||
]);
|
||||
|
||||
export function availableComputerActions(
|
||||
actions: readonly ComputerUseV2ActionName[],
|
||||
hasCleanupOwner: boolean,
|
||||
): readonly ComputerUseV2ActionName[] {
|
||||
return hasCleanupOwner
|
||||
? actions
|
||||
: actions.filter((action) => !EXECUTION_OWNED_ACTIONS.has(action));
|
||||
}
|
||||
|
||||
export function createComputerToolSchema(actions: readonly ComputerUseV2ActionName[]) {
|
||||
return Type.Object({
|
||||
action: stringEnum(actions),
|
||||
...gatewayCallOptionSchemaProperties(),
|
||||
node: Type.Optional(
|
||||
Type.String({
|
||||
description:
|
||||
"Paired node id or display name. Omit when exactly one connected computer-capable node exists.",
|
||||
}),
|
||||
),
|
||||
// Codex accepts a single schema in array `items`, not tuple item arrays.
|
||||
// Fixed bounds preserve the coordinate-pair contract across runtimes.
|
||||
coordinate: Type.Optional(
|
||||
Type.Array(Type.Integer({ minimum: 0 }), {
|
||||
minItems: 2,
|
||||
maxItems: 2,
|
||||
description: "[x, y] target in pixels of the most recent screenshot.",
|
||||
}),
|
||||
),
|
||||
startCoordinate: Type.Optional(
|
||||
Type.Array(Type.Integer({ minimum: 0 }), {
|
||||
minItems: 2,
|
||||
maxItems: 2,
|
||||
description: "left_click_drag: [x, y] drag origin in screenshot pixels.",
|
||||
}),
|
||||
),
|
||||
destinationCoordinate: Type.Optional(
|
||||
Type.Array(Type.Number({ minimum: 0 }), {
|
||||
minItems: 2,
|
||||
maxItems: 2,
|
||||
description: "browser_pointer drag destination [x, y] in viewport CSS pixels.",
|
||||
}),
|
||||
),
|
||||
text: Type.Optional(
|
||||
Type.String({
|
||||
description:
|
||||
'type: text to type; key/hold_key: key combo such as "cmd+shift+t" or "Return"; ' +
|
||||
'click/scroll actions: modifier keys to hold ("shift", "ctrl", "alt", "cmd").',
|
||||
}),
|
||||
),
|
||||
scrollDirection: optionalStringEnum(["up", "down", "left", "right"] as const),
|
||||
scrollAmount: optionalPositiveIntegerSchema({
|
||||
maximum: 100,
|
||||
description: "scroll: number of wheel ticks.",
|
||||
}),
|
||||
duration: optionalFiniteNumberSchema({
|
||||
minimum: 0,
|
||||
maximum: MAX_WAIT_SECONDS,
|
||||
description: `Seconds. hold_key: >0 to ${MAX_HOLD_SECONDS}; wait: 0 to ${MAX_WAIT_SECONDS}.`,
|
||||
}),
|
||||
screenIndex: optionalNonNegativeIntegerSchema(),
|
||||
frameId: Type.Optional(
|
||||
Type.String({
|
||||
description:
|
||||
"Coordinate actions: exact frame id returned by the most recent screenshot result.",
|
||||
}),
|
||||
),
|
||||
windowRef: Type.Optional(
|
||||
Type.String({ description: "Opaque window reference from observation." }),
|
||||
),
|
||||
browserRef: Type.Optional(
|
||||
Type.String({ description: "Opaque browser reference from get_browser_state." }),
|
||||
),
|
||||
pageRef: Type.Optional(
|
||||
Type.String({ description: "Opaque browser page reference from get_browser_state." }),
|
||||
),
|
||||
elementRef: Type.Optional(
|
||||
Type.String({ description: "Opaque accessibility element reference from observation." }),
|
||||
),
|
||||
observationId: Type.Optional(
|
||||
Type.String({ description: "Observation id that issued window or element references." }),
|
||||
),
|
||||
deliveryMode: optionalStringEnum(["background", "foreground"] as const),
|
||||
query: Type.Optional(Type.String()),
|
||||
depth: Type.Optional(Type.Integer({ minimum: 0, maximum: 64 })),
|
||||
maxElements: Type.Optional(Type.Integer({ minimum: 1, maximum: 2_000 })),
|
||||
app: Type.Optional(Type.String()),
|
||||
value: Type.Optional(Type.String()),
|
||||
path: Type.Optional(
|
||||
Type.Array(Type.String({ minLength: 1, maxLength: 200 }), { minItems: 1, maxItems: 16 }),
|
||||
),
|
||||
x1: Type.Optional(Type.Number({ minimum: 0 })),
|
||||
y1: Type.Optional(Type.Number({ minimum: 0 })),
|
||||
x2: Type.Optional(Type.Number({ minimum: 0 })),
|
||||
y2: Type.Optional(Type.Number({ minimum: 0 })),
|
||||
reason: optionalStringEnum([
|
||||
"ax_tree_pixel_mismatch",
|
||||
"background_delivery_failed",
|
||||
"foreground_ineffective",
|
||||
"no_window_target",
|
||||
"other",
|
||||
] as const),
|
||||
snapshotFormat: optionalStringEnum(["dom_refs_v1", "semantic_v2"] as const),
|
||||
continuation: Type.Optional(Type.String()),
|
||||
includeScreenshot: Type.Optional(Type.Boolean()),
|
||||
profile: optionalStringEnum(["isolated_new", "isolated_named"] as const),
|
||||
profileName: Type.Optional(Type.String({ minLength: 1, maxLength: 64 })),
|
||||
url: Type.Optional(Type.String()),
|
||||
inputRoute: optionalStringEnum(["trusted", "dom_event"] as const),
|
||||
mode: optionalStringEnum(["insert_text", "keystrokes"] as const),
|
||||
replace: Type.Optional(Type.Boolean()),
|
||||
dialogAction: optionalStringEnum(["inspect", "accept", "dismiss"] as const),
|
||||
dialogRef: Type.Optional(Type.String()),
|
||||
promptText: Type.Optional(Type.String()),
|
||||
resourceHandle: Type.Optional(
|
||||
Type.String({ description: "Opaque node-owned Computer Use resource handle." }),
|
||||
),
|
||||
resourceHandles: Type.Optional(
|
||||
Type.Array(Type.String({ minLength: 1 }), { minItems: 1, maxItems: 32 }),
|
||||
),
|
||||
recordVideo: Type.Optional(Type.Boolean()),
|
||||
delayMs: Type.Optional(Type.Integer({ minimum: 0, maximum: 10_000 })),
|
||||
stopOnError: Type.Optional(Type.Boolean()),
|
||||
pointerAction: optionalStringEnum([
|
||||
"hover",
|
||||
"right_click",
|
||||
"double_click",
|
||||
"scroll",
|
||||
"drag",
|
||||
] as const),
|
||||
destinationElementRef: Type.Optional(Type.String()),
|
||||
deltaX: Type.Optional(Type.Number()),
|
||||
deltaY: Type.Optional(Type.Number()),
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
import type {
|
||||
ComputerUseCapabilityDescriptor,
|
||||
ComputerUseV2ActionName,
|
||||
} from "../../plugins/computer-use-contract.js";
|
||||
|
||||
export const COMPUTER_ACT_COMMAND = "computer.act";
|
||||
export const SCREEN_SNAPSHOT_COMMAND = "screen.snapshot";
|
||||
|
||||
export const COMPUTER_REF_WIDTH = 1280;
|
||||
export const SCREENSHOT_QUALITY = 0.85;
|
||||
export const AFTER_ACTION_SCREENSHOT_DELAY_MS = 500;
|
||||
export const MAX_WAIT_SECONDS = 100;
|
||||
export const MAX_HOLD_SECONDS = 10;
|
||||
export const MODEL_OBSERVATION_MAX_ELEMENTS = 200;
|
||||
|
||||
export type ComputerToolAction = ComputerUseV2ActionName;
|
||||
|
||||
export type ComputerTarget = { nodeId: string; screenIndex: number };
|
||||
|
||||
export type ComputerFrame = {
|
||||
target: ComputerTarget;
|
||||
id: string;
|
||||
displayFrameId: string;
|
||||
contextEpoch: number;
|
||||
};
|
||||
|
||||
export type ScreenshotCapture = {
|
||||
base64: string;
|
||||
displayFrameId: string;
|
||||
mimeType: string;
|
||||
width?: number;
|
||||
height?: number;
|
||||
};
|
||||
|
||||
export type ComputerObservationState = {
|
||||
nodeId: string;
|
||||
providerGeneration: string;
|
||||
observationId: string;
|
||||
};
|
||||
|
||||
export type ComputerContextEpoch = {
|
||||
value: number;
|
||||
/** Tool result whose screenshot currently authorizes coordinates. */
|
||||
frameToolCallId?: string;
|
||||
/** Digest of the exact sanitized image the model received for that result. */
|
||||
frameImageIdentity?: string;
|
||||
};
|
||||
|
||||
export type ResolvedComputerTarget = {
|
||||
target: ComputerTarget;
|
||||
frame?: ComputerFrame;
|
||||
capabilities?: ComputerUseCapabilityDescriptor;
|
||||
};
|
||||
+122
-1421
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user