mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-27 21:07:01 -06:00
improve(browser): batch navigation guard, snapshot delta markers, tool output schema, code-mode recipe (#113749)
* feat(browser): guard batches and annotate snapshot deltas * docs(browser): document reliable automation loops * test(browser): align batch guard mocks * fix(browser): reset snapshot deltas after navigation * fix(browser): detect document navigation without URL equality * fix(browser): require stable identity for snapshot deltas * fix(browser): keep snapshot delta helper private
This commit is contained in:
committed by
GitHub
parent
0ff2c82033
commit
4dd8a2be76
@@ -328,6 +328,13 @@ OpenClaw supports two "snapshot" styles:
|
||||
- If Playwright is unavailable, ARIA snapshots can still be useful for
|
||||
inspection, but refs may not be actionable. Re-snapshot with `--format ai`
|
||||
or `--interactive` when you need action refs.
|
||||
- When the driver exposes stable document identity, consecutive AI and role
|
||||
snapshots for the same profile, tab, document, and option family append
|
||||
`[new]` to ref-bearing lines absent from the previous snapshot. Navigation
|
||||
starts a fresh unmarked baseline; existing-session snapshots omit deltas.
|
||||
The first snapshot establishes the baseline without markers; later responses
|
||||
also expose `newElements`, and add a count footer when the value is nonzero.
|
||||
Structured `--format aria` snapshots with `axN` refs do not use delta markers.
|
||||
- Docker proof for the raw-CDP fallback path: `pnpm test:docker:browser-cdp-snapshot`
|
||||
starts Chromium with CDP, runs `browser doctor --deep`, and verifies role
|
||||
snapshots include link URLs, cursor-promoted clickables, and iframe metadata.
|
||||
@@ -335,6 +342,10 @@ OpenClaw supports two "snapshot" styles:
|
||||
Ref behavior:
|
||||
|
||||
- Refs are **not stable across navigations**; if something fails, re-run `snapshot` and use a fresh ref.
|
||||
- A batch stops after a committed main-frame navigation—including a same-URL
|
||||
reload—or after the page closes. Its `aborted` summary reports the action
|
||||
number and skipped count; take a fresh snapshot before issuing dependent
|
||||
actions, or use separate act calls when navigation is expected.
|
||||
- `/act` returns the current raw `targetId` after action-triggered replacement
|
||||
when it can prove the replacement tab. Keep using stable tab ids/labels for
|
||||
follow-up commands.
|
||||
|
||||
@@ -197,6 +197,11 @@ extraction mode when a caller does not pass an explicit `snapshotFormat` or
|
||||
`mode`; see [Browser control API](/tools/browser-control) for per-call
|
||||
snapshot options.
|
||||
|
||||
On drivers with stable document identity, repeated AI or role snapshots of the
|
||||
same tab, document, and option family mark newly appeared ref-bearing elements
|
||||
with `[new]`. The first snapshot—and the first snapshot after navigation—sets
|
||||
an unmarked baseline. Existing-session snapshots omit deltas.
|
||||
|
||||
### Tab cleanup ownership
|
||||
|
||||
Session tab cleanup applies only to tabs created by the OpenClaw browser tool
|
||||
|
||||
@@ -43,6 +43,44 @@ Use this skill when you need the `browser` tool for anything beyond a single pag
|
||||
- Target id: nested actions share the request's tab; an explicit nested `targetId` that resolves to a different tab is rejected with `ACT_TARGET_ID_MISMATCH`.
|
||||
- Response: `{ "results": [{ "ok": true } | { "ok": false, "error": "..." }, ...] }` in order; with default `stopOnError` the array ends at the first failure. Any failed entry exits nonzero; use `--json` to preserve the full response in scripts.
|
||||
|
||||
## Code Mode Loop
|
||||
|
||||
When `tools.codeMode` is enabled, call the Browser tool from exec cells:
|
||||
|
||||
```javascript
|
||||
const browserTool = "openclaw:browser:browser";
|
||||
let previousSnapshot = "";
|
||||
const callBrowser = async (input) => await tools.call(browserTool, input);
|
||||
```
|
||||
|
||||
Keep the same labeled tab through the loop, and alternate reads with actions:
|
||||
|
||||
```javascript
|
||||
const snapshotCall = await callBrowser({
|
||||
action: "snapshot",
|
||||
targetId: "task",
|
||||
refs: "aria",
|
||||
interactive: true,
|
||||
});
|
||||
const details = snapshotCall?.result?.details ?? {};
|
||||
const snapshot = (snapshotCall?.result?.content ?? []).map((block) => block?.text ?? "").join("\n");
|
||||
const relevant = snapshot
|
||||
.split("\n")
|
||||
.filter((line) => /submit|dialog|error|\[new\]/i.test(line))
|
||||
.slice(0, 12);
|
||||
const changed = snapshot !== previousSnapshot;
|
||||
previousSnapshot = snapshot;
|
||||
return { targetId: details.targetId, url: details.url, relevant, changed };
|
||||
```
|
||||
|
||||
- Request interactive-only snapshots and filter them in code before returning.
|
||||
- Return only the handful of relevant elements; never return the full tree.
|
||||
- Keep `previousSnapshot` between cells when a local diff helps explain a change.
|
||||
- Interleave each act with a URL or tabs check before the next dependent act.
|
||||
- If a batch returns `aborted`, take a fresh snapshot before continuing.
|
||||
- If `[new]` markers appear, inspect those elements first, then update the saved snapshot.
|
||||
- Use separate act calls when navigation is expected between steps.
|
||||
|
||||
## Tab Hygiene
|
||||
|
||||
Before creating a tab for a named task, list tabs and reuse an existing matching label or URL when it is still usable.
|
||||
|
||||
@@ -13,6 +13,7 @@ export function describeBrowserTool(opts: {
|
||||
"When using refs from snapshot (e.g. e12), keep the same tab: prefer passing targetId from the snapshot response into subsequent actions (act/click/type/etc). For tab operations, targetId also accepts tabId handles (t1) and labels from action=tabs.",
|
||||
"For multi-step browser work, login checks, stale refs, duplicate tabs, or Google Meet flows, use the bundled browser-automation skill when it is available.",
|
||||
'For stable, self-resolving refs across calls, use snapshot with refs="aria" (Playwright aria-ref ids). Default refs="role" are role+name-based.',
|
||||
"Repeated compatible snapshots with stable document identity mark newly appeared ref-bearing elements with [new].",
|
||||
"Use snapshot+act for UI automation. Avoid act:wait by default; use only in exceptional cases when no reliable UI state exists.",
|
||||
"For file chooser uploads, pass the trigger ref with paths in the same upload call when available; use paths-only arming only when a later trigger is intentional. Use inputRef or element to set a file input directly.",
|
||||
`target selects browser location (sandbox|host|node). Default: ${opts.targetDefault}.`,
|
||||
|
||||
@@ -459,6 +459,7 @@ export async function executeSnapshotAction(params: {
|
||||
targetId: snapshot.targetId,
|
||||
url: snapshot.url,
|
||||
truncated: snapshot.truncated,
|
||||
newElements: snapshot.newElements,
|
||||
stats: snapshot.stats,
|
||||
refs: snapshot.refs ? Object.keys(snapshot.refs).length : undefined,
|
||||
labels: snapshot.labels,
|
||||
@@ -646,7 +647,7 @@ export async function executeActAction(params: {
|
||||
readStringValue((result as { targetId?: unknown }).targetId) ??
|
||||
readStringValue(effectiveRequest.targetId),
|
||||
);
|
||||
return jsonResult(result);
|
||||
return formatActToolResult(result);
|
||||
} catch (err) {
|
||||
if (isChromeStaleTargetError(profile, err)) {
|
||||
const tabs = proxyRequest
|
||||
@@ -688,7 +689,7 @@ export async function executeActAction(params: {
|
||||
readStringValue((retryResult as { targetId?: unknown }).targetId) ??
|
||||
readStringValue(retryRequest.targetId),
|
||||
);
|
||||
return jsonResult(retryResult);
|
||||
return formatActToolResult(retryResult);
|
||||
}
|
||||
if (!tabs.length) {
|
||||
throw new Error(
|
||||
@@ -704,3 +705,37 @@ export async function executeActAction(params: {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
function formatActToolResult(result: unknown): AgentToolResult<unknown> {
|
||||
const formatted = jsonResult(result);
|
||||
if (!result || typeof result !== "object") {
|
||||
return formatted;
|
||||
}
|
||||
const aborted = (result as { aborted?: unknown }).aborted;
|
||||
if (!aborted || typeof aborted !== "object") {
|
||||
return formatted;
|
||||
}
|
||||
const summary = aborted as {
|
||||
reason?: unknown;
|
||||
afterAction?: unknown;
|
||||
url?: unknown;
|
||||
skipped?: unknown;
|
||||
};
|
||||
if (
|
||||
(summary.reason !== "navigation" && summary.reason !== "closed") ||
|
||||
typeof summary.afterAction !== "number" ||
|
||||
typeof summary.url !== "string" ||
|
||||
typeof summary.skipped !== "number"
|
||||
) {
|
||||
return formatted;
|
||||
}
|
||||
const reason =
|
||||
summary.reason === "navigation"
|
||||
? `the page navigated to ${summary.url}`
|
||||
: "the page or browser context closed";
|
||||
const note = `Batch aborted after action ${summary.afterAction} because ${reason}; ${summary.skipped} remaining action(s) skipped. Take a new snapshot before continuing.`;
|
||||
return {
|
||||
...formatted,
|
||||
content: [...formatted.content, { type: "text", text: note }],
|
||||
};
|
||||
}
|
||||
|
||||
@@ -33,7 +33,7 @@ export {
|
||||
normalizeOptionalString,
|
||||
readStringValue,
|
||||
} from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
export { BrowserToolSchema } from "./browser-tool.schema.js";
|
||||
export { BrowserToolOutputSchema, BrowserToolSchema } from "./browser-tool.schema.js";
|
||||
export {
|
||||
browserAct,
|
||||
browserArmDialog,
|
||||
|
||||
@@ -177,3 +177,78 @@ export const BrowserToolSchema = Type.Object({
|
||||
fn: Type.Optional(Type.String()),
|
||||
request: Type.Optional(BrowserActSchema),
|
||||
});
|
||||
|
||||
const BrowserSnapshotStatsSchema = Type.Object(
|
||||
{
|
||||
lines: Type.Number(),
|
||||
chars: Type.Number(),
|
||||
refs: Type.Number(),
|
||||
interactive: Type.Number(),
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
);
|
||||
|
||||
const BrowserBatchAbortSchema = Type.Object(
|
||||
{
|
||||
reason: stringEnum(["navigation", "closed"] as const),
|
||||
afterAction: Type.Number(),
|
||||
url: Type.String(),
|
||||
skipped: Type.Number(),
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
);
|
||||
|
||||
/** Common structured result fields returned across Browser tool actions. */
|
||||
export const BrowserToolOutputSchema = Type.Object(
|
||||
{
|
||||
ok: Type.Optional(Type.Boolean()),
|
||||
targetId: Type.Optional(Type.String()),
|
||||
url: Type.Optional(Type.String()),
|
||||
format: Type.Optional(stringEnum(BROWSER_SNAPSHOT_FORMATS)),
|
||||
snapshot: Type.Optional(Type.String()),
|
||||
refs: Type.Optional(Type.Union([Type.Number(), Type.Record(Type.String(), Type.Unknown())])),
|
||||
stats: Type.Optional(BrowserSnapshotStatsSchema),
|
||||
truncated: Type.Optional(Type.Boolean()),
|
||||
newElements: Type.Optional(Type.Number()),
|
||||
tabs: Type.Optional(
|
||||
Type.Array(
|
||||
Type.Object(
|
||||
{
|
||||
suggestedTargetId: Type.Optional(Type.String()),
|
||||
tabId: Type.Optional(Type.String()),
|
||||
label: Type.Optional(Type.String()),
|
||||
targetId: Type.Optional(Type.String()),
|
||||
title: Type.Optional(Type.String()),
|
||||
url: Type.Optional(Type.String()),
|
||||
type: Type.Optional(Type.String()),
|
||||
},
|
||||
{ additionalProperties: true },
|
||||
),
|
||||
),
|
||||
),
|
||||
tabCount: Type.Optional(Type.Number()),
|
||||
results: Type.Optional(
|
||||
Type.Array(
|
||||
Type.Object(
|
||||
{
|
||||
ok: Type.Boolean(),
|
||||
error: Type.Optional(Type.String()),
|
||||
navigated: Type.Optional(Type.Literal(true)),
|
||||
url: Type.Optional(Type.String()),
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
),
|
||||
),
|
||||
),
|
||||
aborted: Type.Optional(BrowserBatchAbortSchema),
|
||||
enabled: Type.Optional(Type.Boolean()),
|
||||
running: Type.Optional(Type.Boolean()),
|
||||
profile: Type.Optional(Type.String()),
|
||||
driver: Type.Optional(Type.String()),
|
||||
transport: Type.Optional(Type.String()),
|
||||
pid: Type.Optional(Type.Union([Type.Number(), Type.Null()])),
|
||||
cdpPort: Type.Optional(Type.Union([Type.Number(), Type.Null()])),
|
||||
cdpUrl: Type.Optional(Type.Union([Type.String(), Type.Null()])),
|
||||
},
|
||||
{ additionalProperties: true },
|
||||
);
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
// Browser tests cover browser tool plugin behavior.
|
||||
import { Value } from "typebox/value";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const browserClientMocks = vi.hoisted(() => ({
|
||||
@@ -55,7 +56,7 @@ const browserClientMocks = vi.hoisted(() => ({
|
||||
vi.mock("./browser/client.js", () => browserClientMocks);
|
||||
|
||||
const browserActionsMocks = vi.hoisted(() => ({
|
||||
browserAct: vi.fn(async () => ({ ok: true })),
|
||||
browserAct: vi.fn(async (): Promise<Record<string, unknown>> => ({ ok: true })),
|
||||
browserArmDialog: vi.fn(async () => ({ ok: true })),
|
||||
browserArmFileChooser: vi.fn(async () => ({ ok: true })),
|
||||
browserConsoleMessages: vi.fn(async () => ({
|
||||
@@ -213,7 +214,10 @@ vi.mock("./sdk-setup-tools.js", async () => {
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("./browser-tool.runtime.js", () => {
|
||||
vi.mock("./browser-tool.runtime.js", async () => {
|
||||
const { BrowserToolOutputSchema } = await vi.importActual<
|
||||
typeof import("./browser-tool.schema.js")
|
||||
>("./browser-tool.schema.js");
|
||||
const readStringValue = (value: unknown) => (typeof value === "string" ? value : undefined);
|
||||
const readStringParam = (
|
||||
params: Record<string, unknown>,
|
||||
@@ -233,6 +237,7 @@ vi.mock("./browser-tool.runtime.js", () => {
|
||||
return {
|
||||
DEFAULT_AI_SNAPSHOT_MAX_CHARS: 40_000,
|
||||
DEFAULT_UPLOAD_DIR: "/tmp/openclaw-browser-uploads",
|
||||
BrowserToolOutputSchema,
|
||||
BrowserToolSchema: {},
|
||||
...browserActionsMocks,
|
||||
...browserClientMocks,
|
||||
@@ -487,6 +492,20 @@ function lastNodeInvokeCall(): ReturnType<typeof nodeInvokeCall> {
|
||||
return nodeInvokeCall(-1);
|
||||
}
|
||||
|
||||
describe("browser tool output schema", () => {
|
||||
it("accepts snapshot details", async () => {
|
||||
const tool = createBrowserTool();
|
||||
const result = await tool.execute?.("call-1", {
|
||||
action: "snapshot",
|
||||
target: "host",
|
||||
snapshotFormat: "ai",
|
||||
});
|
||||
|
||||
expect(tool.outputSchema).toBeDefined();
|
||||
expect(Value.Check(tool.outputSchema!, result?.details)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("browser tool description", () => {
|
||||
it("warns agents about existing-session act timeout limits", () => {
|
||||
const tool = createBrowserTool();
|
||||
@@ -2257,6 +2276,39 @@ describe("browser tool url alias support", () => {
|
||||
describe("browser tool act compatibility", () => {
|
||||
registerBrowserToolAfterEachReset();
|
||||
|
||||
it("adds a clear note when a batch aborts after navigation", async () => {
|
||||
browserActionsMocks.browserAct.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
results: [{ ok: true, navigated: true, url: "https://example.com/next" }],
|
||||
aborted: {
|
||||
reason: "navigation",
|
||||
afterAction: 1,
|
||||
url: "https://example.com/next",
|
||||
skipped: 2,
|
||||
},
|
||||
});
|
||||
const tool = createBrowserTool();
|
||||
|
||||
const result = await tool.execute?.("call-1", {
|
||||
action: "act",
|
||||
request: {
|
||||
kind: "batch",
|
||||
actions: [
|
||||
{ kind: "click", ref: "1" },
|
||||
{ kind: "click", ref: "2" },
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
expect(result?.details).toMatchObject({ aborted: { reason: "navigation", skipped: 2 } });
|
||||
expect(result?.content.at(-1)).toMatchObject({
|
||||
type: "text",
|
||||
text: expect.stringContaining(
|
||||
"Batch aborted after action 1 because the page navigated to https://example.com/next",
|
||||
),
|
||||
});
|
||||
});
|
||||
|
||||
it("accepts flattened act params for backward compatibility", async () => {
|
||||
const tool = createBrowserTool();
|
||||
await tool.execute?.("call-1", {
|
||||
|
||||
@@ -21,6 +21,7 @@ import {
|
||||
import {
|
||||
type AnyAgentTool,
|
||||
type NodeListNode,
|
||||
BrowserToolOutputSchema,
|
||||
BrowserToolSchema,
|
||||
browserAct,
|
||||
browserArmDialog,
|
||||
@@ -418,6 +419,7 @@ export function createBrowserTool(opts?: {
|
||||
name: "browser",
|
||||
description: describeBrowserTool({ targetDefault, hostHint }),
|
||||
parameters: BrowserToolSchema,
|
||||
outputSchema: BrowserToolOutputSchema,
|
||||
execute: async (_toolCallId, args) => {
|
||||
const bindingResult =
|
||||
opts?.runToolBinding === undefined
|
||||
|
||||
@@ -1,9 +1,18 @@
|
||||
// Browser tests cover CDP committed page-session URL observation.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { prepareCdpTargetSession } from "./cdp-page-session.js";
|
||||
import { prepareCdpTargetSession, readCdpMainFrameDocumentIdentity } from "./cdp-page-session.js";
|
||||
import type { CdpSendFn } from "./cdp.helpers.js";
|
||||
|
||||
describe("prepareCdpTargetSession", () => {
|
||||
it("reads the main-frame loader identity", async () => {
|
||||
const send: CdpSendFn = async (method) =>
|
||||
method === "Page.getFrameTree"
|
||||
? { frameTree: { frame: { loaderId: "LOADER_SAME_URL" } } }
|
||||
: {};
|
||||
|
||||
await expect(readCdpMainFrameDocumentIdentity(send)).resolves.toBe("cdp:LOADER_SAME_URL");
|
||||
});
|
||||
|
||||
it("ignores Chrome's transient colon URL while navigation is settling", async () => {
|
||||
let frameReadCount = 0;
|
||||
const send: CdpSendFn = async (method) => {
|
||||
|
||||
@@ -42,6 +42,18 @@ function readCommittedFrameUrl(
|
||||
return url ? `${url}${fragment}` : undefined;
|
||||
}
|
||||
|
||||
/** Read the browser-owned loader identity for the committed main-frame document. */
|
||||
export async function readCdpMainFrameDocumentIdentity(
|
||||
send: CdpSendFn,
|
||||
sessionId?: string,
|
||||
): Promise<string | undefined> {
|
||||
const frameTree = (await send("Page.getFrameTree", undefined, sessionId).catch(
|
||||
() => null,
|
||||
)) as CdpFrameTreeResult | null;
|
||||
const loaderId = frameTree?.frameTree?.frame?.loaderId;
|
||||
return typeof loaderId === "string" && loaderId.trim() ? `cdp:${loaderId.trim()}` : undefined;
|
||||
}
|
||||
|
||||
async function waitForCdpNavigationResult(
|
||||
send: CdpSendFn,
|
||||
sessionId: string | undefined,
|
||||
|
||||
@@ -11,6 +11,7 @@ import type { SsrFPolicy } from "../infra/net/ssrf.js";
|
||||
import {
|
||||
prepareCdpPageSession,
|
||||
prepareCdpTargetSession,
|
||||
readCdpMainFrameDocumentIdentity,
|
||||
type CdpActionTimeouts,
|
||||
} from "./cdp-page-session.js";
|
||||
import {
|
||||
@@ -26,12 +27,24 @@ import {
|
||||
withCdpSocket,
|
||||
} from "./cdp.helpers.js";
|
||||
import { assertBrowserNavigationAllowed, withBrowserNavigationPolicy } from "./navigation-guard.js";
|
||||
import { finalizeRoleSnapshot } from "./pw-role-snapshot.js";
|
||||
import { finalizeRoleSnapshot, type RoleSnapshotIdentityMode } from "./pw-role-snapshot.js";
|
||||
import { CONTENT_ROLES, INTERACTIVE_ROLES, STRUCTURAL_ROLES } from "./snapshot-roles.js";
|
||||
|
||||
export { appendCdpPath } from "./cdp.helpers.js";
|
||||
export { type CdpActionTimeouts, waitForCdpCommittedNavigationUrl } from "./cdp-page-session.js";
|
||||
|
||||
/** Read the current main-frame loader identity from a page-level CDP target. */
|
||||
export async function getMainFrameDocumentIdentityViaCdp(opts: {
|
||||
wsUrl: string;
|
||||
timeoutMs?: number;
|
||||
}): Promise<string | undefined> {
|
||||
return await withCdpSocket(
|
||||
opts.wsUrl,
|
||||
async (send) => await readCdpMainFrameDocumentIdentity(send),
|
||||
{ commandTimeoutMs: opts.timeoutMs ?? 5000 },
|
||||
);
|
||||
}
|
||||
|
||||
/** Normalize a reported CDP WebSocket URL against the configured CDP base URL. */
|
||||
export function normalizeCdpWsUrl(wsUrl: string, cdpUrl: string): string {
|
||||
const ws = new URL(wsUrl);
|
||||
@@ -891,11 +904,13 @@ export async function snapshotRoleViaCdp(opts: {
|
||||
urls?: boolean;
|
||||
timeoutMs?: number;
|
||||
maxChars?: number;
|
||||
delta?: { mode: RoleSnapshotIdentityMode; previousKeys?: ReadonlySet<string> };
|
||||
}): Promise<{
|
||||
snapshot: string;
|
||||
truncated?: boolean;
|
||||
refs: Record<string, CdpRoleRef>;
|
||||
stats: { lines: number; chars: number; refs: number; interactive: number };
|
||||
newElements?: number;
|
||||
}> {
|
||||
return await withCdpSocket(
|
||||
opts.wsUrl,
|
||||
@@ -915,6 +930,7 @@ export async function snapshotRoleViaCdp(opts: {
|
||||
snapshot,
|
||||
refs: built.refs,
|
||||
maxChars: opts.maxChars,
|
||||
delta: opts.delta,
|
||||
});
|
||||
},
|
||||
{ commandTimeoutMs: opts.timeoutMs ?? 5000 },
|
||||
|
||||
@@ -17,6 +17,8 @@ import type {
|
||||
BrowserActionOk,
|
||||
BrowserActionPathResult,
|
||||
BrowserActionTabResult,
|
||||
BrowserBatchAbort,
|
||||
BrowserBatchActionResult,
|
||||
} from "./client-actions-types.js";
|
||||
import { buildProfileQuery, withBaseUrl } from "./client-actions-url.js";
|
||||
import type { BrowserActRequest } from "./client-actions.types.js";
|
||||
@@ -34,7 +36,8 @@ type BrowserActResponse = {
|
||||
targetId: string;
|
||||
url?: string;
|
||||
result?: unknown;
|
||||
results?: Array<{ ok: boolean; error?: string }>;
|
||||
results?: BrowserBatchActionResult[];
|
||||
aborted?: BrowserBatchAbort;
|
||||
blockedByDialog?: boolean;
|
||||
browserState?: unknown;
|
||||
/** Download info when a click/batch/evaluate action triggers a browser download. */
|
||||
|
||||
@@ -7,6 +7,22 @@ import type { AnnotationItem } from "./screenshot-annotate.js";
|
||||
/** Generic success result for action endpoints. */
|
||||
export type BrowserActionOk = { ok: true };
|
||||
|
||||
/** Per-action result returned by a browser batch. */
|
||||
export type BrowserBatchActionResult = {
|
||||
ok: boolean;
|
||||
error?: string;
|
||||
navigated?: true;
|
||||
url?: string;
|
||||
};
|
||||
|
||||
/** Summary returned when a batch cannot safely continue on its original page. */
|
||||
export type BrowserBatchAbort = {
|
||||
reason: "navigation" | "closed";
|
||||
afterAction: number;
|
||||
url: string;
|
||||
skipped: number;
|
||||
};
|
||||
|
||||
/** Success result carrying the affected tab and optional URL. */
|
||||
export type BrowserActionTabResult = {
|
||||
ok: true;
|
||||
|
||||
@@ -140,6 +140,7 @@ export type SnapshotResult =
|
||||
url: string;
|
||||
snapshot: string;
|
||||
truncated?: boolean;
|
||||
newElements?: number;
|
||||
refs?: Record<string, { role: string; name?: string; nth?: number }>;
|
||||
stats?: {
|
||||
lines: number;
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
forceDisconnectPlaywrightForTarget,
|
||||
getObservedBrowserStateForPage,
|
||||
getObservedBrowserStateViaPlaywright,
|
||||
getMainFrameDocumentIdentityViaPlaywright,
|
||||
getPageForTargetId,
|
||||
isBrowserObservedDialogBlockedError,
|
||||
listPagesViaPlaywright,
|
||||
@@ -92,6 +93,7 @@ export const pwAi = {
|
||||
createObservedDialogAbortSignalForPage,
|
||||
getObservedBrowserStateForPage,
|
||||
getObservedBrowserStateViaPlaywright,
|
||||
getMainFrameDocumentIdentityViaPlaywright,
|
||||
getPageForTargetId,
|
||||
isBrowserObservedDialogBlockedError,
|
||||
listPagesViaPlaywright,
|
||||
|
||||
@@ -4,6 +4,7 @@ import {
|
||||
buildRoleSnapshotFromAiSnapshot,
|
||||
buildRoleSnapshotFromAriaSnapshot,
|
||||
finalizeRoleSnapshot,
|
||||
getRoleSnapshotIdentityKeys,
|
||||
parseRoleRef,
|
||||
} from "./pw-role-snapshot.js";
|
||||
|
||||
@@ -131,6 +132,75 @@ describe("pw-role-snapshot", () => {
|
||||
expect(result.refs).toEqual({ e1: { role: "button" } });
|
||||
});
|
||||
|
||||
it("does not mark the first snapshot", () => {
|
||||
const snapshot = '- button "Save" [ref=e1]';
|
||||
const refs = { e1: { role: "button", name: "Save" } };
|
||||
|
||||
const result = finalizeRoleSnapshot({ snapshot, refs, delta: { mode: "role" } });
|
||||
|
||||
expect(result.snapshot).toBe(snapshot);
|
||||
expect(result.newElements).toBeUndefined();
|
||||
});
|
||||
|
||||
it("marks only new role identities and preserves ref extraction", () => {
|
||||
const previousKeys = getRoleSnapshotIdentityKeys(
|
||||
{ e1: { role: "button", name: "Save" } },
|
||||
"role",
|
||||
);
|
||||
const refs = {
|
||||
e7: { role: "button", name: "Save" },
|
||||
e8: { role: "dialog", name: "Confirmation" },
|
||||
};
|
||||
const finalized = finalizeRoleSnapshot({
|
||||
snapshot: ['- button "Save" [ref=e7]', '- dialog "Confirmation" [ref=e8]'].join("\n"),
|
||||
refs,
|
||||
delta: { mode: "role", previousKeys },
|
||||
});
|
||||
|
||||
expect(finalized.snapshot).toBe(
|
||||
[
|
||||
'- button "Save" [ref=e7]',
|
||||
'- dialog "Confirmation" [ref=e8] [new]',
|
||||
"1 new element(s) since last snapshot",
|
||||
].join("\n"),
|
||||
);
|
||||
expect(finalized.newElements).toBe(1);
|
||||
expect(finalized.refs).toEqual(refs);
|
||||
});
|
||||
|
||||
it("uses preserved aria refs as AI snapshot identities", () => {
|
||||
const finalized = finalizeRoleSnapshot({
|
||||
snapshot: ['- button "Save" [ref=7]', '- dialog "Confirmation" [ref=8]'].join("\n"),
|
||||
refs: {
|
||||
"7": { role: "button", name: "Save" },
|
||||
"8": { role: "dialog", name: "Confirmation" },
|
||||
},
|
||||
delta: { mode: "aria", previousKeys: new Set(["7"]) },
|
||||
});
|
||||
|
||||
expect(finalized.snapshot).toContain('- button "Save" [ref=7]\n');
|
||||
expect(finalized.snapshot).toContain('- dialog "Confirmation" [ref=8] [new]');
|
||||
expect(finalized.newElements).toBe(1);
|
||||
});
|
||||
|
||||
it("annotates before truncation and keeps only complete annotated refs", () => {
|
||||
const first = '- button "Visible" [ref=e1] [new]';
|
||||
const marker = "[...TRUNCATED - page too large]";
|
||||
const result = finalizeRoleSnapshot({
|
||||
snapshot: ['- button "Visible" [ref=e1]', '- dialog "Hidden" [ref=e2]'].join("\n"),
|
||||
refs: {
|
||||
e1: { role: "button", name: "Visible" },
|
||||
e2: { role: "dialog", name: "Hidden" },
|
||||
},
|
||||
maxChars: first.length + 2 + marker.length,
|
||||
delta: { mode: "role", previousKeys: new Set() },
|
||||
});
|
||||
|
||||
expect(result.snapshot).toBe(`${first}\n\n${marker}`);
|
||||
expect(result.refs).toEqual({ e1: { role: "button", name: "Visible" } });
|
||||
expect(result.newElements).toBe(1);
|
||||
});
|
||||
|
||||
it("treats sub-unit internal budgets as uncapped", () => {
|
||||
const snapshot = '- button "Visible" [ref=e1]';
|
||||
const result = finalizeRoleSnapshot({
|
||||
|
||||
@@ -18,6 +18,9 @@ type RoleRef = {
|
||||
/** Mapping from generated role refs to role/name metadata. */
|
||||
export type RoleRefMap = Record<string, RoleRef>;
|
||||
|
||||
/** Identity strategy used to compare consecutive ref-bearing snapshots. */
|
||||
export type RoleSnapshotIdentityMode = "role" | "aria";
|
||||
|
||||
type RoleSnapshotStats = {
|
||||
lines: number;
|
||||
chars: number;
|
||||
@@ -57,6 +60,54 @@ function findSnapshotLineRef(line: string): string | undefined {
|
||||
return ROLE_SNAPSHOT_LINE_REF_RE.exec(line)?.[1];
|
||||
}
|
||||
|
||||
/** Build the stable identity set used for per-tab snapshot deltas. */
|
||||
export function getRoleSnapshotIdentityKeys<T extends RoleRef>(
|
||||
refs: Record<string, T>,
|
||||
mode: RoleSnapshotIdentityMode,
|
||||
): Set<string> {
|
||||
// Duplicate role+name elements are identified positionally by nth, so insertion can mark a
|
||||
// sibling duplicate. This is acceptable: they are actor-indistinguishable without DOM backing.
|
||||
return new Set(
|
||||
Object.entries(refs).map(([ref, value]) =>
|
||||
mode === "aria" ? ref : `${value.role}\0${value.name ?? ""}\0${value.nth ?? 0}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
/** Mark ref-bearing lines that were absent from the previous compatible snapshot. */
|
||||
function annotateRoleSnapshotDelta<T extends RoleRef>(params: {
|
||||
snapshot: string;
|
||||
refs: Record<string, T>;
|
||||
mode: RoleSnapshotIdentityMode;
|
||||
previousKeys?: ReadonlySet<string>;
|
||||
}): { snapshot: string; keys: Set<string>; newElements?: number } {
|
||||
const keys = getRoleSnapshotIdentityKeys(params.refs, params.mode);
|
||||
if (params.previousKeys === undefined) {
|
||||
return { snapshot: params.snapshot, keys };
|
||||
}
|
||||
const keyByRef = new Map(
|
||||
Object.entries(params.refs).map(([ref, value]) => [
|
||||
ref,
|
||||
params.mode === "aria" ? ref : `${value.role}\0${value.name ?? ""}\0${value.nth ?? 0}`,
|
||||
]),
|
||||
);
|
||||
const markedKeys = new Set<string>();
|
||||
const lines = params.snapshot.split("\n").map((line) => {
|
||||
const ref = findSnapshotLineRef(line);
|
||||
const key = ref ? keyByRef.get(ref) : undefined;
|
||||
if (!key || params.previousKeys?.has(key)) {
|
||||
return line;
|
||||
}
|
||||
markedKeys.add(key);
|
||||
return `${line} [new]`;
|
||||
});
|
||||
const newElements = markedKeys.size;
|
||||
if (newElements > 0) {
|
||||
lines.push(`${newElements} new element(s) since last snapshot`);
|
||||
}
|
||||
return { snapshot: lines.join("\n"), keys, newElements };
|
||||
}
|
||||
|
||||
function truncateRoleSnapshot(snapshot: string, maxChars: number): string {
|
||||
const marker =
|
||||
maxChars >= ROLE_SNAPSHOT_TRUNCATION_MARKER.length ? ROLE_SNAPSHOT_TRUNCATION_MARKER : "…";
|
||||
@@ -72,23 +123,37 @@ function truncateRoleSnapshot(snapshot: string, maxChars: number): string {
|
||||
}
|
||||
|
||||
/** Apply the final output budget, then keep only refs present on complete output lines. */
|
||||
export function finalizeRoleSnapshot<T extends { role: string }>(params: {
|
||||
export function finalizeRoleSnapshot<T extends RoleRef>(params: {
|
||||
snapshot: string;
|
||||
refs: Record<string, T>;
|
||||
maxChars?: number;
|
||||
delta?: {
|
||||
mode: RoleSnapshotIdentityMode;
|
||||
previousKeys?: ReadonlySet<string>;
|
||||
};
|
||||
}): {
|
||||
snapshot: string;
|
||||
truncated?: boolean;
|
||||
refs: Record<string, T>;
|
||||
stats: RoleSnapshotStats;
|
||||
newElements?: number;
|
||||
} {
|
||||
const normalizedMaxChars =
|
||||
typeof params.maxChars === "number" && Number.isFinite(params.maxChars) && params.maxChars > 0
|
||||
? Math.floor(params.maxChars)
|
||||
: undefined;
|
||||
const maxChars = normalizedMaxChars && normalizedMaxChars > 0 ? normalizedMaxChars : undefined;
|
||||
const truncated = maxChars !== undefined && params.snapshot.length > maxChars;
|
||||
const snapshot = truncated ? truncateRoleSnapshot(params.snapshot, maxChars) : params.snapshot;
|
||||
const annotated = params.delta
|
||||
? annotateRoleSnapshotDelta({
|
||||
snapshot: params.snapshot,
|
||||
refs: params.refs,
|
||||
mode: params.delta.mode,
|
||||
previousKeys: params.delta.previousKeys,
|
||||
})
|
||||
: undefined;
|
||||
const sourceSnapshot = annotated?.snapshot ?? params.snapshot;
|
||||
const truncated = maxChars !== undefined && sourceSnapshot.length > maxChars;
|
||||
const snapshot = truncated ? truncateRoleSnapshot(sourceSnapshot, maxChars) : sourceSnapshot;
|
||||
const visibleRefs = new Set(
|
||||
snapshot
|
||||
.split("\n")
|
||||
@@ -98,10 +163,17 @@ export function finalizeRoleSnapshot<T extends { role: string }>(params: {
|
||||
const refs = Object.fromEntries(
|
||||
Object.entries(params.refs).filter(([ref]) => visibleRefs.has(ref)),
|
||||
) as Record<string, T>;
|
||||
const newElements =
|
||||
params.delta?.previousKeys === undefined
|
||||
? undefined
|
||||
: [...getRoleSnapshotIdentityKeys(refs, params.delta.mode)].filter(
|
||||
(key) => !params.delta?.previousKeys?.has(key),
|
||||
).length;
|
||||
const result = {
|
||||
snapshot,
|
||||
refs,
|
||||
stats: getRoleSnapshotStats(snapshot, refs),
|
||||
...(newElements !== undefined ? { newElements } : {}),
|
||||
};
|
||||
return truncated ? { ...result, truncated: true } : result;
|
||||
}
|
||||
|
||||
@@ -52,7 +52,10 @@ import {
|
||||
getObservedBrowserStateForPage,
|
||||
normalizeCdpUrl,
|
||||
} from "./pw-session-state.js";
|
||||
import { BROWSER_REF_MARKER_ATTRIBUTE } from "./pw-session.page-cdp.js";
|
||||
import {
|
||||
BROWSER_REF_MARKER_ATTRIBUTE,
|
||||
readMainFrameDocumentIdentityForPage,
|
||||
} from "./pw-session.page-cdp.js";
|
||||
|
||||
export async function getObservedBrowserStateViaPlaywright(opts: {
|
||||
cdpUrl: string;
|
||||
@@ -63,6 +66,15 @@ export async function getObservedBrowserStateViaPlaywright(opts: {
|
||||
return getObservedBrowserStateForPage(page);
|
||||
}
|
||||
|
||||
/** Resolve a page and read its current main-frame document identity. */
|
||||
export async function getMainFrameDocumentIdentityViaPlaywright(opts: {
|
||||
cdpUrl: string;
|
||||
targetId?: string;
|
||||
}): Promise<string | undefined> {
|
||||
const page = await getPageForTargetId(opts);
|
||||
return await readMainFrameDocumentIdentityForPage(page);
|
||||
}
|
||||
|
||||
export function refLocator(page: Page, ref: string) {
|
||||
const normalized = ref.startsWith("@")
|
||||
? ref.slice(1)
|
||||
|
||||
@@ -3,6 +3,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
BROWSER_REF_MARKER_ATTRIBUTE,
|
||||
markBackendDomRefsOnPage,
|
||||
readMainFrameDocumentIdentityForPage,
|
||||
withPageScopedCdpClient,
|
||||
} from "./pw-session.page-cdp.js";
|
||||
|
||||
@@ -38,6 +39,25 @@ describe("pw-session page-scoped CDP client", () => {
|
||||
expect(sessionDetach).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("reads the main-frame loader identity through the existing page session", async () => {
|
||||
const sessionSend = vi.fn(async (method: string) =>
|
||||
method === "Page.getFrameTree"
|
||||
? { frameTree: { frame: { loaderId: "LOADER_SAME_URL" } } }
|
||||
: {},
|
||||
);
|
||||
const sessionDetach = vi.fn(async () => {});
|
||||
const page = {
|
||||
context: () => ({
|
||||
newCDPSession: vi.fn(async () => ({ send: sessionSend, detach: sessionDetach })),
|
||||
}),
|
||||
};
|
||||
|
||||
await expect(readMainFrameDocumentIdentityForPage(page as never)).resolves.toBe(
|
||||
"cdp:LOADER_SAME_URL",
|
||||
);
|
||||
expect(sessionDetach).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("marks backend DOM refs on the page", async () => {
|
||||
const sessionSend = vi.fn(async (method: string, params?: Record<string, unknown>) => {
|
||||
if (method === "DOM.pushNodesByBackendIdsToFrontend") {
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
*/
|
||||
import { uniqueValues } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import type { CDPSession, Page } from "playwright-core";
|
||||
import { readCdpMainFrameDocumentIdentity } from "./cdp-page-session.js";
|
||||
|
||||
type PageCdpSend = (method: string, params?: Record<string, unknown>) => Promise<unknown>;
|
||||
type MarkBackendDomRef = { ref: string; backendDOMNodeId: number };
|
||||
@@ -44,6 +45,24 @@ export async function withPageScopedCdpClient<T>(opts: {
|
||||
});
|
||||
}
|
||||
|
||||
/** Read the browser-owned loader identity for a Playwright page's main frame. */
|
||||
export async function readMainFrameDocumentIdentityForPage(
|
||||
page: Page,
|
||||
): Promise<string | undefined> {
|
||||
return await withPlaywrightPageCdpSession(
|
||||
page,
|
||||
async (session) =>
|
||||
await readCdpMainFrameDocumentIdentity((method, params) =>
|
||||
(
|
||||
session.send as unknown as (
|
||||
method: string,
|
||||
params?: Record<string, unknown>,
|
||||
) => Promise<unknown>
|
||||
)(method, params),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
/** Mark backend DOM node ids on the page with browser ref attributes. */
|
||||
export async function markBackendDomRefsOnPage(opts: {
|
||||
page: Page;
|
||||
|
||||
@@ -41,6 +41,7 @@ export {
|
||||
createPageViaPlaywright,
|
||||
focusPageByTargetIdViaPlaywright,
|
||||
forceDisconnectPlaywrightForTarget,
|
||||
getMainFrameDocumentIdentityViaPlaywright,
|
||||
getObservedBrowserStateViaPlaywright,
|
||||
listPagesViaPlaywright,
|
||||
refLocator,
|
||||
|
||||
@@ -4,10 +4,16 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
let page: {
|
||||
evaluate: ReturnType<typeof vi.fn>;
|
||||
keyboard: { press: ReturnType<typeof vi.fn> };
|
||||
isClosed: ReturnType<typeof vi.fn>;
|
||||
mainFrame: ReturnType<typeof vi.fn>;
|
||||
mouse: { click: ReturnType<typeof vi.fn> };
|
||||
off: ReturnType<typeof vi.fn>;
|
||||
on: ReturnType<typeof vi.fn>;
|
||||
url: ReturnType<typeof vi.fn>;
|
||||
} | null = null;
|
||||
let locator: Record<string, ReturnType<typeof vi.fn>> | null = null;
|
||||
let setPageClosed: (closed: boolean) => void = () => {};
|
||||
let setPageUrl: (url: string) => void = () => {};
|
||||
|
||||
const getPageForTargetId = vi.fn(async () => {
|
||||
if (!page) {
|
||||
@@ -19,6 +25,7 @@ const ensurePageState = vi.fn(() => {});
|
||||
const assertPageNavigationCompletedSafely = vi.fn(async () => {});
|
||||
const forceDisconnectPlaywrightForTarget = vi.fn(async () => {});
|
||||
const isBrowserObservedDialogBlockedError = vi.fn(() => false);
|
||||
const isPolicyDenyNavigationError = vi.fn(() => false);
|
||||
const markObservedDialogsHandledRemotelyForPage = vi.fn(() => ({}));
|
||||
const refLocator = vi.fn(() => {
|
||||
if (!locator) {
|
||||
@@ -47,6 +54,7 @@ vi.mock("./pw-session.js", () => ({
|
||||
forceDisconnectPlaywrightForTarget,
|
||||
getPageForTargetId,
|
||||
isBrowserObservedDialogBlockedError,
|
||||
isPolicyDenyNavigationError,
|
||||
markObservedDialogsHandledRemotelyForPage,
|
||||
refLocator,
|
||||
restoreRoleRefsForTarget,
|
||||
@@ -76,25 +84,204 @@ describe("batchViaPlaywright", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
let currentUrl = "https://example.com";
|
||||
const navigate = vi.fn(async () => {
|
||||
currentUrl = "https://93.184.216.34/target";
|
||||
});
|
||||
let closed = false;
|
||||
setPageClosed = (next) => {
|
||||
closed = next;
|
||||
};
|
||||
const frameNavigatedHandlers = new Set<(frame: unknown) => void>();
|
||||
const mainFrame = { url: () => currentUrl };
|
||||
setPageUrl = (next) => {
|
||||
currentUrl = next;
|
||||
for (const handler of frameNavigatedHandlers) {
|
||||
handler(mainFrame);
|
||||
}
|
||||
};
|
||||
page = {
|
||||
evaluate: navigate,
|
||||
keyboard: { press: navigate },
|
||||
mouse: { click: navigate },
|
||||
evaluate: vi.fn(async () => {}),
|
||||
isClosed: vi.fn(() => closed),
|
||||
keyboard: { press: vi.fn(async () => {}) },
|
||||
mainFrame: vi.fn(() => mainFrame),
|
||||
mouse: { click: vi.fn(async () => {}) },
|
||||
off: vi.fn((event: string, handler: (frame: unknown) => void) => {
|
||||
if (event === "framenavigated") {
|
||||
frameNavigatedHandlers.delete(handler);
|
||||
}
|
||||
}),
|
||||
on: vi.fn((event: string, handler: (frame: unknown) => void) => {
|
||||
if (event === "framenavigated") {
|
||||
frameNavigatedHandlers.add(handler);
|
||||
}
|
||||
}),
|
||||
url: vi.fn(() => currentUrl),
|
||||
};
|
||||
locator = {
|
||||
click: navigate,
|
||||
dragTo: navigate,
|
||||
fill: navigate,
|
||||
hover: navigate,
|
||||
press: navigate,
|
||||
scrollIntoViewIfNeeded: navigate,
|
||||
selectOption: navigate,
|
||||
setChecked: navigate,
|
||||
click: vi.fn(async () => {}),
|
||||
dragTo: vi.fn(async () => {}),
|
||||
fill: vi.fn(async () => {}),
|
||||
hover: vi.fn(async () => {}),
|
||||
press: vi.fn(async () => {}),
|
||||
scrollIntoViewIfNeeded: vi.fn(async () => {}),
|
||||
selectOption: vi.fn(async () => {}),
|
||||
setChecked: vi.fn(async () => {}),
|
||||
};
|
||||
closePageViaPlaywright.mockImplementation(async () => setPageClosed(true));
|
||||
});
|
||||
|
||||
it("aborts remaining actions after a navigation", async () => {
|
||||
locator!.click!.mockImplementationOnce(() => {
|
||||
setPageUrl("https://example.com/next");
|
||||
});
|
||||
|
||||
const result = await batchViaPlaywright({
|
||||
cdpUrl: "http://127.0.0.1:9222",
|
||||
targetId: "tab-1",
|
||||
actions: [
|
||||
{ kind: "click", ref: "1" },
|
||||
{ kind: "hover", ref: "2" },
|
||||
{ kind: "press", key: "Enter" },
|
||||
],
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
results: [{ ok: true, navigated: true, url: "https://example.com/next" }],
|
||||
aborted: {
|
||||
reason: "navigation",
|
||||
afterAction: 1,
|
||||
url: "https://example.com/next",
|
||||
skipped: 2,
|
||||
},
|
||||
});
|
||||
expect(locator!.hover).not.toHaveBeenCalled();
|
||||
expect(page!.keyboard.press).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("aborts remaining actions after a same-URL reload", async () => {
|
||||
locator!.click!.mockImplementationOnce(() => {
|
||||
setPageUrl("https://example.com");
|
||||
});
|
||||
|
||||
const result = await batchViaPlaywright({
|
||||
cdpUrl: "http://127.0.0.1:9222",
|
||||
targetId: "tab-1",
|
||||
actions: [
|
||||
{ kind: "click", ref: "1" },
|
||||
{ kind: "hover", ref: "2" },
|
||||
],
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
results: [{ ok: true, navigated: true, url: "https://example.com" }],
|
||||
aborted: {
|
||||
reason: "navigation",
|
||||
afterAction: 1,
|
||||
url: "https://example.com",
|
||||
skipped: 1,
|
||||
},
|
||||
});
|
||||
expect(locator!.hover).not.toHaveBeenCalled();
|
||||
expect(page!.off).toHaveBeenCalledWith("framenavigated", expect.any(Function));
|
||||
});
|
||||
|
||||
it("aborts when a navigation commits after an action settles but before the next dispatch", async () => {
|
||||
let closedChecks = 0;
|
||||
page!.isClosed.mockImplementation(() => {
|
||||
closedChecks += 1;
|
||||
if (closedChecks === 2) {
|
||||
setPageUrl("https://example.com/late");
|
||||
}
|
||||
return false;
|
||||
});
|
||||
|
||||
const result = await batchViaPlaywright({
|
||||
cdpUrl: "http://127.0.0.1:9222",
|
||||
targetId: "tab-1",
|
||||
actions: [
|
||||
{ kind: "click", ref: "1" },
|
||||
{ kind: "hover", ref: "2" },
|
||||
],
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
results: [{ ok: true, navigated: true, url: "https://example.com/late" }],
|
||||
aborted: {
|
||||
reason: "navigation",
|
||||
afterAction: 1,
|
||||
url: "https://example.com/late",
|
||||
skipped: 1,
|
||||
},
|
||||
});
|
||||
expect(locator!.hover).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("runs every action when the page URL stays unchanged", async () => {
|
||||
const result = await batchViaPlaywright({
|
||||
cdpUrl: "http://127.0.0.1:9222",
|
||||
targetId: "tab-1",
|
||||
actions: [
|
||||
{ kind: "click", ref: "1" },
|
||||
{ kind: "hover", ref: "2" },
|
||||
{ kind: "press", key: "Enter" },
|
||||
],
|
||||
});
|
||||
|
||||
expect(result).toEqual({ results: [{ ok: true }, { ok: true }, { ok: true }] });
|
||||
});
|
||||
|
||||
it("keeps stopOnError=false until a later navigation aborts the batch", async () => {
|
||||
locator!.click!.mockRejectedValueOnce(new Error("click failed"));
|
||||
locator!.hover!.mockImplementationOnce(() => {
|
||||
setPageUrl("https://example.com/next");
|
||||
});
|
||||
|
||||
const result = await batchViaPlaywright({
|
||||
cdpUrl: "http://127.0.0.1:9222",
|
||||
targetId: "tab-1",
|
||||
stopOnError: false,
|
||||
actions: [
|
||||
{ kind: "click", ref: "1" },
|
||||
{ kind: "hover", ref: "2" },
|
||||
{ kind: "press", key: "Enter" },
|
||||
],
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
results: [
|
||||
{ ok: false, error: "click failed" },
|
||||
{ ok: true, navigated: true, url: "https://example.com/next" },
|
||||
],
|
||||
aborted: {
|
||||
reason: "navigation",
|
||||
afterAction: 2,
|
||||
url: "https://example.com/next",
|
||||
skipped: 1,
|
||||
},
|
||||
});
|
||||
expect(page!.keyboard.press).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("aborts when the page closes during an action", async () => {
|
||||
locator!.click!.mockImplementationOnce(() => setPageClosed(true));
|
||||
|
||||
const result = await batchViaPlaywright({
|
||||
cdpUrl: "http://127.0.0.1:9222",
|
||||
targetId: "tab-1",
|
||||
stopOnError: false,
|
||||
actions: [
|
||||
{ kind: "click", ref: "1" },
|
||||
{ kind: "hover", ref: "2" },
|
||||
],
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
results: [{ ok: true }],
|
||||
aborted: {
|
||||
reason: "closed",
|
||||
afterAction: 1,
|
||||
url: "https://example.com",
|
||||
skipped: 1,
|
||||
},
|
||||
});
|
||||
expect(locator!.hover).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("propagates evaluate timeouts through batched execution", async () => {
|
||||
|
||||
@@ -16,6 +16,7 @@ import {
|
||||
resolveActInteractionTimeoutMs,
|
||||
resolveActWaitTimeoutMs,
|
||||
} from "./act-policy.js";
|
||||
import type { BrowserBatchAbort, BrowserBatchActionResult } from "./client-actions-types.js";
|
||||
import type { BrowserActRequest, BrowserFormField } from "./client-actions.types.js";
|
||||
import type { BrowserDownloadResult } from "./download-types.js";
|
||||
import { normalizeBrowserEvaluateFunctionSource } from "./evaluate-source.js";
|
||||
@@ -1953,7 +1954,8 @@ export async function executeActViaPlaywright(
|
||||
} & BrowserNavigationPolicyOptions,
|
||||
): Promise<{
|
||||
result?: unknown;
|
||||
results?: Array<{ ok: boolean; error?: string }>;
|
||||
results?: BrowserBatchActionResult[];
|
||||
aborted?: BrowserBatchAbort;
|
||||
blockedByDialog?: boolean;
|
||||
browserState?: unknown;
|
||||
downloads?: BrowserDownloadResult[];
|
||||
@@ -1995,6 +1997,7 @@ export async function executeActViaPlaywright(
|
||||
const batch = await batchViaPlaywright({
|
||||
cdpUrl: opts.cdpUrl,
|
||||
targetId: opts.targetId,
|
||||
page,
|
||||
...navigationPolicy,
|
||||
actions: opts.action.actions,
|
||||
stopOnError: opts.action.stopOnError,
|
||||
@@ -2004,6 +2007,7 @@ export async function executeActViaPlaywright(
|
||||
const newDownloads = await drainDownloads();
|
||||
return {
|
||||
results: batch.results,
|
||||
...(batch.aborted ? { aborted: batch.aborted } : {}),
|
||||
...(newDownloads ? { downloads: newDownloads } : {}),
|
||||
};
|
||||
}
|
||||
@@ -2059,13 +2063,14 @@ export async function batchViaPlaywright(
|
||||
opts: {
|
||||
cdpUrl: string;
|
||||
targetId?: string;
|
||||
page?: Page;
|
||||
actions: BrowserActRequest[];
|
||||
stopOnError?: boolean;
|
||||
evaluateEnabled?: boolean;
|
||||
depth?: number;
|
||||
signal?: AbortSignal;
|
||||
} & BrowserNavigationPolicyOptions,
|
||||
): Promise<{ results: Array<{ ok: boolean; error?: string }> }> {
|
||||
): Promise<{ results: BrowserBatchActionResult[]; aborted?: BrowserBatchAbort }> {
|
||||
const navigationPolicy = interactionNavigationPolicy(opts);
|
||||
const depth = opts.depth ?? 0;
|
||||
if (depth > ACT_MAX_BATCH_DEPTH) {
|
||||
@@ -2074,37 +2079,101 @@ export async function batchViaPlaywright(
|
||||
if (opts.actions.length > ACT_MAX_BATCH_ACTIONS) {
|
||||
throw new Error(`Batch exceeds maximum of ${ACT_MAX_BATCH_ACTIONS} actions`);
|
||||
}
|
||||
const results: Array<{ ok: boolean; error?: string }> = [];
|
||||
for (const action of opts.actions) {
|
||||
if (opts.signal?.aborted) {
|
||||
throw opts.signal.reason ?? new Error("aborted");
|
||||
const page = opts.page ?? (await getPageForTargetId(opts));
|
||||
const results: BrowserBatchActionResult[] = [];
|
||||
const finishAborted = (
|
||||
reason: BrowserBatchAbort["reason"],
|
||||
afterAction: number,
|
||||
url: string,
|
||||
skipped: number,
|
||||
) =>
|
||||
skipped === 0
|
||||
? { results }
|
||||
: { results, aborted: { reason, afterAction, url, skipped } satisfies BrowserBatchAbort };
|
||||
let mainFrameNavigations = 0;
|
||||
let navigationsAtLastDispatch = 0;
|
||||
const currentMainFrameUrl = () => page.mainFrame?.().url() ?? page.url();
|
||||
const onFrameNavigated = (frame: Frame) => {
|
||||
if (frame === page.mainFrame?.()) {
|
||||
mainFrameNavigations += 1;
|
||||
}
|
||||
try {
|
||||
await executeSingleAction(
|
||||
action,
|
||||
opts.cdpUrl,
|
||||
opts.targetId,
|
||||
opts.evaluateEnabled,
|
||||
navigationPolicy,
|
||||
depth,
|
||||
opts.signal,
|
||||
);
|
||||
results.push({ ok: true });
|
||||
} catch (err) {
|
||||
if (isBrowserObservedDialogBlockedError(err)) {
|
||||
throw err;
|
||||
};
|
||||
const finishNavigation = (afterAction: number, skipped: number) => {
|
||||
const url = currentMainFrameUrl();
|
||||
const lastResult = results.at(-1);
|
||||
if (lastResult) {
|
||||
results[results.length - 1] = { ...lastResult, navigated: true, url };
|
||||
}
|
||||
return finishAborted("navigation", afterAction, url, skipped);
|
||||
};
|
||||
|
||||
// Snapshot refs are document-scoped, so any committed main-frame navigation
|
||||
// ends the batch. A commit after the next action dispatch is inherently unguardable;
|
||||
// callers that expect navigation can use separate act calls as the escape hatch.
|
||||
page.on?.("framenavigated", onFrameNavigated);
|
||||
try {
|
||||
for (const [index, action] of opts.actions.entries()) {
|
||||
if (opts.signal?.aborted) {
|
||||
throw opts.signal.reason ?? new Error("aborted");
|
||||
}
|
||||
if (isPolicyDenyNavigationError(err)) {
|
||||
throw err;
|
||||
if (mainFrameNavigations > navigationsAtLastDispatch) {
|
||||
return finishNavigation(index, opts.actions.length - index);
|
||||
}
|
||||
const message = formatErrorMessage(err);
|
||||
results.push({ ok: false, error: message });
|
||||
if (opts.stopOnError !== false) {
|
||||
break;
|
||||
if (page.isClosed?.()) {
|
||||
return finishAborted("closed", index, currentMainFrameUrl(), opts.actions.length - index);
|
||||
}
|
||||
navigationsAtLastDispatch = mainFrameNavigations;
|
||||
try {
|
||||
await executeSingleAction(
|
||||
action,
|
||||
opts.cdpUrl,
|
||||
opts.targetId,
|
||||
opts.evaluateEnabled,
|
||||
navigationPolicy,
|
||||
depth,
|
||||
opts.signal,
|
||||
);
|
||||
results.push({ ok: true });
|
||||
if (page.isClosed?.()) {
|
||||
return finishAborted(
|
||||
"closed",
|
||||
index + 1,
|
||||
currentMainFrameUrl(),
|
||||
opts.actions.length - index - 1,
|
||||
);
|
||||
}
|
||||
if (mainFrameNavigations > navigationsAtLastDispatch) {
|
||||
return finishNavigation(index + 1, opts.actions.length - index - 1);
|
||||
}
|
||||
} catch (err) {
|
||||
if (isBrowserObservedDialogBlockedError(err)) {
|
||||
throw err;
|
||||
}
|
||||
if (isPolicyDenyNavigationError(err)) {
|
||||
throw err;
|
||||
}
|
||||
const message = formatErrorMessage(err);
|
||||
results.push({ ok: false, error: message });
|
||||
if (page.isClosed?.()) {
|
||||
return finishAborted(
|
||||
"closed",
|
||||
index + 1,
|
||||
currentMainFrameUrl(),
|
||||
opts.actions.length - index - 1,
|
||||
);
|
||||
}
|
||||
if (mainFrameNavigations > navigationsAtLastDispatch) {
|
||||
return finishNavigation(index + 1, opts.actions.length - index - 1);
|
||||
}
|
||||
if (opts.stopOnError !== false) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return { results };
|
||||
} finally {
|
||||
page.off?.("framenavigated", onFrameNavigated);
|
||||
}
|
||||
return { results };
|
||||
}
|
||||
|
||||
/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */
|
||||
|
||||
@@ -24,6 +24,7 @@ import {
|
||||
buildRoleSnapshotFromAiSnapshot,
|
||||
buildRoleSnapshotFromAriaSnapshot,
|
||||
finalizeRoleSnapshot,
|
||||
type RoleSnapshotIdentityMode,
|
||||
type RoleSnapshotOptions,
|
||||
type RoleRefMap,
|
||||
} from "./pw-role-snapshot.js";
|
||||
@@ -260,7 +261,13 @@ export async function snapshotAiViaPlaywright(opts: {
|
||||
maxChars?: number;
|
||||
urls?: boolean;
|
||||
ssrfPolicy?: SsrFPolicy;
|
||||
}): Promise<{ snapshot: string; truncated?: boolean; refs: RoleRefMap }> {
|
||||
delta?: { mode: RoleSnapshotIdentityMode; previousKeys?: ReadonlySet<string> };
|
||||
}): Promise<{
|
||||
snapshot: string;
|
||||
truncated?: boolean;
|
||||
refs: RoleRefMap;
|
||||
newElements?: number;
|
||||
}> {
|
||||
const page = await prepareSnapshotPageViaPlaywright({
|
||||
cdpUrl: opts.cdpUrl,
|
||||
targetId: opts.targetId,
|
||||
@@ -282,6 +289,7 @@ export async function snapshotAiViaPlaywright(opts: {
|
||||
snapshot,
|
||||
refs: built.refs,
|
||||
maxChars: opts.maxChars,
|
||||
delta: opts.delta,
|
||||
});
|
||||
assertSnapshotFrameCurrent(isFrameCurrent);
|
||||
storeRoleRefsForTarget({
|
||||
@@ -335,11 +343,13 @@ async function finalizeRoleSnapshotViaPlaywright(params: {
|
||||
built: { snapshot: string; refs: RoleRefMap };
|
||||
urls?: boolean;
|
||||
maxChars?: number;
|
||||
delta?: { mode: RoleSnapshotIdentityMode; previousKeys?: ReadonlySet<string> };
|
||||
}): Promise<{
|
||||
snapshot: string;
|
||||
truncated?: boolean;
|
||||
refs: RoleRefMap;
|
||||
stats: { lines: number; chars: number; refs: number; interactive: number };
|
||||
newElements?: number;
|
||||
}> {
|
||||
const snapshot = params.urls
|
||||
? appendSnapshotUrls(params.built.snapshot, await collectSnapshotUrls(params.page))
|
||||
@@ -351,6 +361,7 @@ async function finalizeRoleSnapshotViaPlaywright(params: {
|
||||
snapshot,
|
||||
refs: params.built.refs,
|
||||
maxChars: params.maxChars,
|
||||
delta: params.delta,
|
||||
});
|
||||
storeRoleRefsForTarget({
|
||||
page: params.page,
|
||||
@@ -376,11 +387,13 @@ export async function snapshotRoleViaPlaywright(opts: {
|
||||
maxChars?: number;
|
||||
timeoutMs?: number;
|
||||
ssrfPolicy?: SsrFPolicy;
|
||||
delta?: { mode: RoleSnapshotIdentityMode; previousKeys?: ReadonlySet<string> };
|
||||
}): Promise<{
|
||||
snapshot: string;
|
||||
truncated?: boolean;
|
||||
refs: Record<string, { role: string; name?: string; nth?: number }>;
|
||||
stats: { lines: number; chars: number; refs: number; interactive: number };
|
||||
newElements?: number;
|
||||
}> {
|
||||
const page = await prepareSnapshotPageViaPlaywright({
|
||||
cdpUrl: opts.cdpUrl,
|
||||
@@ -411,6 +424,7 @@ export async function snapshotRoleViaPlaywright(opts: {
|
||||
mode: "aria",
|
||||
urls: opts.urls,
|
||||
maxChars: opts.maxChars,
|
||||
delta: opts.delta,
|
||||
});
|
||||
},
|
||||
});
|
||||
@@ -456,6 +470,7 @@ export async function snapshotRoleViaPlaywright(opts: {
|
||||
mode: "role",
|
||||
urls: opts.urls,
|
||||
maxChars: opts.maxChars,
|
||||
delta: opts.delta,
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
@@ -31,6 +31,7 @@ import {
|
||||
} from "../navigation-guard.js";
|
||||
import { getBrowserProfileCapabilities } from "../profile-capabilities.js";
|
||||
import type { BrowserRouteContext } from "../server-context.js";
|
||||
import { clearSnapshotKeysForTab } from "../snapshot-delta-cache.js";
|
||||
import { matchBrowserUrlPattern } from "../url-pattern.js";
|
||||
import { registerBrowserAgentActDownloadRoutes } from "./agent.act.download.js";
|
||||
import {
|
||||
@@ -691,6 +692,7 @@ export function registerBrowserAgentActRoutes(
|
||||
...existingSessionCallOptions,
|
||||
exactTargetId: true,
|
||||
});
|
||||
clearSnapshotKeysForTab(ctx, profileCtx.profile.name, tab.targetId);
|
||||
return await jsonOk();
|
||||
case "batch":
|
||||
return jsonActError(
|
||||
@@ -721,10 +723,17 @@ export function registerBrowserAgentActRoutes(
|
||||
});
|
||||
}
|
||||
const downloads = result.downloads;
|
||||
if (action.kind === "close" || result.aborted?.reason === "closed") {
|
||||
clearSnapshotKeysForTab(ctx, profileCtx.profile.name, tab.targetId);
|
||||
}
|
||||
switch (action.kind) {
|
||||
case "batch":
|
||||
return await jsonOk(
|
||||
{ results: result.results ?? [], ...(downloads ? { downloads } : {}) },
|
||||
{
|
||||
results: result.results ?? [],
|
||||
...(result.aborted ? { aborted: result.aborted } : {}),
|
||||
...(downloads ? { downloads } : {}),
|
||||
},
|
||||
{ resolveCurrentTarget: true },
|
||||
);
|
||||
case "evaluate":
|
||||
|
||||
@@ -206,6 +206,35 @@ describe("existing-session browser routes", () => {
|
||||
expect(chromeMcpMocks.takeChromeMcpScreenshot).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("omits deltas for existing-session snapshots without stable document identity", async () => {
|
||||
chromeMcpMocks.takeChromeMcpSnapshot
|
||||
.mockResolvedValueOnce({
|
||||
id: "root-1",
|
||||
role: "document",
|
||||
name: "Example",
|
||||
children: [{ id: "save-1", role: "button", name: "Save" }],
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
id: "root-2",
|
||||
role: "document",
|
||||
name: "Example",
|
||||
children: [
|
||||
{ id: "save-2", role: "button", name: "Save" },
|
||||
{ id: "alert-2", role: "alert", name: "Required" },
|
||||
],
|
||||
});
|
||||
const handler = getSnapshotGetHandler();
|
||||
const first = createBrowserRouteResponse();
|
||||
const second = createBrowserRouteResponse();
|
||||
|
||||
await handler?.({ params: {}, query: { format: "ai" } }, first.res);
|
||||
await handler?.({ params: {}, query: { format: "ai" } }, second.res);
|
||||
|
||||
const body = requireRecord(second.body, "second snapshot body");
|
||||
expect(body.snapshot).not.toContain("[new]");
|
||||
expect(body.newElements).toBeUndefined();
|
||||
});
|
||||
|
||||
it("labels and returns only Chrome MCP refs inside the final snapshot budget", async () => {
|
||||
chromeMcpMocks.takeChromeMcpSnapshot.mockResolvedValueOnce({
|
||||
id: "root",
|
||||
|
||||
@@ -20,10 +20,13 @@ const routeState = vi.hoisted(() => ({
|
||||
}));
|
||||
|
||||
const cdpMocks = vi.hoisted(() => ({
|
||||
getMainFrameDocumentIdentityViaCdp: vi.fn<() => Promise<string | undefined>>(
|
||||
async () => "cdp:test-document",
|
||||
),
|
||||
snapshotAria: vi.fn(async () => ({
|
||||
nodes: [{ ref: "1", role: "link", name: "private", depth: 0 }],
|
||||
})),
|
||||
snapshotRoleViaCdp: vi.fn(async () => ({
|
||||
snapshotRoleViaCdp: vi.fn(async (_opts: unknown) => ({
|
||||
snapshot: '- link "private" [ref=e1]',
|
||||
refs: { e1: { role: "link", name: "private" } },
|
||||
stats: { lines: 1, chars: 25, refs: 1, interactive: 1 },
|
||||
@@ -40,6 +43,7 @@ const navigationGuardMocks = vi.hoisted(() => ({
|
||||
|
||||
vi.mock("../cdp.js", () => ({
|
||||
captureScreenshot: vi.fn(),
|
||||
getMainFrameDocumentIdentityViaCdp: cdpMocks.getMainFrameDocumentIdentityViaCdp,
|
||||
snapshotAria: cdpMocks.snapshotAria,
|
||||
snapshotRoleViaCdp: cdpMocks.snapshotRoleViaCdp,
|
||||
}));
|
||||
@@ -115,6 +119,7 @@ function getSnapshotGetHandler() {
|
||||
describe("local-managed browser snapshot routes", () => {
|
||||
beforeEach(() => {
|
||||
routeState.profileCtx.ensureTabAvailable.mockClear();
|
||||
cdpMocks.getMainFrameDocumentIdentityViaCdp.mockReset().mockResolvedValue("cdp:test-document");
|
||||
cdpMocks.snapshotAria.mockClear();
|
||||
cdpMocks.snapshotRoleViaCdp.mockClear();
|
||||
navigationGuardMocks.assertBrowserNavigationResultAllowed.mockClear();
|
||||
@@ -171,4 +176,53 @@ describe("local-managed browser snapshot routes", () => {
|
||||
expect.objectContaining({ maxChars: 123 }),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects a snapshot when the main-frame loader changes during capture", async () => {
|
||||
navigationGuardMocks.assertBrowserNavigationResultAllowed.mockResolvedValueOnce(undefined);
|
||||
cdpMocks.getMainFrameDocumentIdentityViaCdp
|
||||
.mockResolvedValueOnce("cdp:before")
|
||||
.mockResolvedValueOnce("cdp:after");
|
||||
const handler = getSnapshotGetHandler();
|
||||
const response = createBrowserRouteResponse();
|
||||
|
||||
await handler?.({ params: {}, query: { format: "ai", interactive: "true" } }, response.res);
|
||||
|
||||
expect(response.statusCode).toBe(400);
|
||||
expect(response.body).toEqual({
|
||||
error: "Frame changed while its browser snapshot was being captured; retry.",
|
||||
});
|
||||
});
|
||||
|
||||
it("disables deltas when no stable document identity is available", async () => {
|
||||
navigationGuardMocks.assertBrowserNavigationResultAllowed.mockResolvedValue(undefined);
|
||||
cdpMocks.getMainFrameDocumentIdentityViaCdp.mockResolvedValue(undefined);
|
||||
const handler = getSnapshotGetHandler();
|
||||
const first = createBrowserRouteResponse();
|
||||
const second = createBrowserRouteResponse();
|
||||
|
||||
await handler?.({ params: {}, query: { format: "ai", interactive: "true" } }, first.res);
|
||||
await handler?.({ params: {}, query: { format: "ai", interactive: "true" } }, second.res);
|
||||
|
||||
const calls = cdpMocks.snapshotRoleViaCdp.mock.calls;
|
||||
expect(calls).toHaveLength(2);
|
||||
expect(calls[0]?.[0]).toMatchObject({ delta: undefined });
|
||||
expect(calls[1]?.[0]).toMatchObject({ delta: undefined });
|
||||
expect(second.body).not.toHaveProperty("newElements");
|
||||
expect(second.body).not.toHaveProperty("snapshot", expect.stringContaining("[new]"));
|
||||
});
|
||||
|
||||
it("reuses delta keys when the stable document identity is unchanged", async () => {
|
||||
navigationGuardMocks.assertBrowserNavigationResultAllowed.mockResolvedValue(undefined);
|
||||
const handler = getSnapshotGetHandler();
|
||||
const first = createBrowserRouteResponse();
|
||||
const second = createBrowserRouteResponse();
|
||||
|
||||
await handler?.({ params: {}, query: { format: "ai", interactive: "true" } }, first.res);
|
||||
await handler?.({ params: {}, query: { format: "ai", interactive: "true" } }, second.res);
|
||||
|
||||
const secondCall = cdpMocks.snapshotRoleViaCdp.mock.calls[1]?.[0];
|
||||
expect(secondCall).toMatchObject({
|
||||
delta: { mode: "role", previousKeys: expect.any(Set) },
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -4,6 +4,7 @@ import { createBrowserRouteApp, createBrowserRouteResponse } from "./test-helper
|
||||
|
||||
const cdpMocks = vi.hoisted(() => ({
|
||||
captureScreenshot: vi.fn(),
|
||||
getMainFrameDocumentIdentityViaCdp: vi.fn(async () => "cdp:test-document"),
|
||||
snapshotAria: vi.fn(async () => ({ nodes: [] })),
|
||||
snapshotRoleViaCdp: vi.fn(async () => ({
|
||||
snapshot: "button Continue",
|
||||
@@ -33,6 +34,7 @@ const profileContext = vi.hoisted(() => ({
|
||||
|
||||
vi.mock("../cdp.js", () => ({
|
||||
captureScreenshot: cdpMocks.captureScreenshot,
|
||||
getMainFrameDocumentIdentityViaCdp: cdpMocks.getMainFrameDocumentIdentityViaCdp,
|
||||
snapshotAria: cdpMocks.snapshotAria,
|
||||
snapshotRoleViaCdp: cdpMocks.snapshotRoleViaCdp,
|
||||
}));
|
||||
|
||||
@@ -8,7 +8,12 @@ import path from "node:path";
|
||||
import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime";
|
||||
import { getImageMetadata } from "../../media/media-services.js";
|
||||
import { ensureMediaDir, saveMediaBuffer } from "../../media/store.js";
|
||||
import { captureScreenshot, snapshotAria, snapshotRoleViaCdp } from "../cdp.js";
|
||||
import {
|
||||
captureScreenshot,
|
||||
getMainFrameDocumentIdentityViaCdp,
|
||||
snapshotAria,
|
||||
snapshotRoleViaCdp,
|
||||
} from "../cdp.js";
|
||||
import {
|
||||
evaluateChromeMcpScript,
|
||||
navigateChromeMcpPage,
|
||||
@@ -27,7 +32,7 @@ import {
|
||||
assertBrowserNavigationResultAllowed,
|
||||
} from "../navigation-guard.js";
|
||||
import { getBrowserProfileCapabilities } from "../profile-capabilities.js";
|
||||
import { finalizeRoleSnapshot } from "../pw-role-snapshot.js";
|
||||
import { finalizeRoleSnapshot, type RoleRefMap } from "../pw-role-snapshot.js";
|
||||
import type { AnnotationItem } from "../screenshot-annotate.js";
|
||||
import { scaleAnnotations } from "../screenshot-annotate.js";
|
||||
import {
|
||||
@@ -36,6 +41,11 @@ import {
|
||||
normalizeBrowserScreenshot,
|
||||
} from "../screenshot.js";
|
||||
import type { BrowserRouteContext } from "../server-context.js";
|
||||
import {
|
||||
getPreviousSnapshotKeys,
|
||||
recordSnapshotKeys,
|
||||
type SnapshotDeltaFamily,
|
||||
} from "../snapshot-delta-cache.js";
|
||||
import { appendSnapshotUrls, type SnapshotUrlEntry } from "../snapshot-urls.js";
|
||||
import { normalizeBrowserTimerDelayMs } from "../timer-delay.js";
|
||||
import {
|
||||
@@ -597,16 +607,56 @@ export function registerBrowserAgentSnapshotRoutes(
|
||||
...ssrfPolicyOpts,
|
||||
});
|
||||
}
|
||||
let observedBrowserState: unknown;
|
||||
if (!usesChromeMcp && pwModule) {
|
||||
observedBrowserState = await pwModule
|
||||
.getObservedBrowserStateViaPlaywright({
|
||||
cdpUrl: profileCtx.profile.cdpUrl,
|
||||
targetId: tab.targetId,
|
||||
ssrfPolicy: ctx.state().resolved.ssrfPolicy,
|
||||
})
|
||||
.catch(() => undefined);
|
||||
}
|
||||
const deltaFamily: SnapshotDeltaFamily | undefined =
|
||||
plan.format === "ai"
|
||||
? {
|
||||
identity: usesChromeMcp
|
||||
? "aria"
|
||||
: plan.wantsRoleSnapshot
|
||||
? plan.refsMode === "aria"
|
||||
? "aria"
|
||||
: "role"
|
||||
: pwModule
|
||||
? "aria"
|
||||
: "role",
|
||||
interactive: plan.interactive,
|
||||
compact: plan.compact,
|
||||
depth: plan.depth,
|
||||
selector: plan.selectorValue,
|
||||
frame: plan.frameSelectorValue,
|
||||
urls: plan.urls,
|
||||
maxChars: plan.resolvedMaxChars,
|
||||
}
|
||||
: undefined;
|
||||
const createDeltaState = (documentIdentity?: string) => {
|
||||
const previousKeys =
|
||||
deltaFamily && documentIdentity
|
||||
? getPreviousSnapshotKeys(ctx, {
|
||||
profile: profileCtx.profile.name,
|
||||
targetId: tab.targetId,
|
||||
documentIdentity,
|
||||
family: deltaFamily,
|
||||
})
|
||||
: undefined;
|
||||
return {
|
||||
delta:
|
||||
deltaFamily && previousKeys !== undefined
|
||||
? { mode: deltaFamily.identity, previousKeys }
|
||||
: undefined,
|
||||
record: (refs: RoleRefMap) => {
|
||||
if (!deltaFamily || !documentIdentity) {
|
||||
return;
|
||||
}
|
||||
recordSnapshotKeys(ctx, {
|
||||
profile: profileCtx.profile.name,
|
||||
targetId: tab.targetId,
|
||||
documentIdentity,
|
||||
family: deltaFamily,
|
||||
refs,
|
||||
});
|
||||
},
|
||||
};
|
||||
};
|
||||
if (usesChromeMcp) {
|
||||
const operation: ChromeMcpSnapshotOperation = {
|
||||
profileName: profileCtx.profile.name,
|
||||
@@ -625,6 +675,7 @@ export function registerBrowserAgentSnapshotRoutes(
|
||||
nodes: flattenChromeMcpSnapshotToAriaNodes(snapshot, plan.limit),
|
||||
});
|
||||
}
|
||||
const deltaState = createDeltaState();
|
||||
const built = buildAiSnapshotFromChromeMcpSnapshot({
|
||||
root: snapshot,
|
||||
options: {
|
||||
@@ -645,6 +696,7 @@ export function registerBrowserAgentSnapshotRoutes(
|
||||
const finalized = finalizeRoleSnapshot({
|
||||
...builtWithUrls,
|
||||
maxChars: plan.resolvedMaxChars,
|
||||
delta: deltaState.delta,
|
||||
});
|
||||
if (plan.labels) {
|
||||
const refs = Object.keys(finalized.refs);
|
||||
@@ -668,6 +720,7 @@ export function registerBrowserAgentSnapshotRoutes(
|
||||
"browser",
|
||||
DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
|
||||
);
|
||||
deltaState.record(finalized.refs);
|
||||
return res.json({
|
||||
ok: true,
|
||||
format: "ai",
|
||||
@@ -684,6 +737,7 @@ export function registerBrowserAgentSnapshotRoutes(
|
||||
await clearChromeMcpOverlay(operation);
|
||||
}
|
||||
}
|
||||
deltaState.record(finalized.refs);
|
||||
return res.json({
|
||||
ok: true,
|
||||
format: "ai",
|
||||
@@ -692,6 +746,18 @@ export function registerBrowserAgentSnapshotRoutes(
|
||||
...finalized,
|
||||
});
|
||||
}
|
||||
const readPlaywrightDocumentIdentity =
|
||||
pwModule?.getMainFrameDocumentIdentityViaPlaywright;
|
||||
let observedBrowserState: unknown;
|
||||
if (pwModule) {
|
||||
observedBrowserState = await pwModule
|
||||
.getObservedBrowserStateViaPlaywright({
|
||||
cdpUrl: profileCtx.profile.cdpUrl,
|
||||
targetId: tab.targetId,
|
||||
ssrfPolicy: ctx.state().resolved.ssrfPolicy,
|
||||
})
|
||||
.catch(() => undefined);
|
||||
}
|
||||
if (hasPendingDialogs(observedBrowserState)) {
|
||||
return res.json({
|
||||
ok: true,
|
||||
@@ -703,6 +769,37 @@ export function registerBrowserAgentSnapshotRoutes(
|
||||
...(plan.format === "aria" ? { nodes: [] } : { snapshot: "", refs: {} }),
|
||||
});
|
||||
}
|
||||
const readDocumentIdentity = async (): Promise<string | undefined> => {
|
||||
if (!deltaFamily) {
|
||||
return undefined;
|
||||
}
|
||||
const playwrightIdentity = readPlaywrightDocumentIdentity
|
||||
? await readPlaywrightDocumentIdentity({
|
||||
cdpUrl: profileCtx.profile.cdpUrl,
|
||||
targetId: tab.targetId,
|
||||
}).catch(() => undefined)
|
||||
: undefined;
|
||||
if (playwrightIdentity || !tab.wsUrl) {
|
||||
return playwrightIdentity;
|
||||
}
|
||||
return await getMainFrameDocumentIdentityViaCdp({
|
||||
wsUrl: tab.wsUrl,
|
||||
timeoutMs: plan.timeoutMs,
|
||||
}).catch(() => undefined);
|
||||
};
|
||||
const initialDocumentIdentity = await readDocumentIdentity();
|
||||
const deltaState = createDeltaState(initialDocumentIdentity);
|
||||
const assertDocumentIdentityUnchanged = async () => {
|
||||
if (!initialDocumentIdentity) {
|
||||
return;
|
||||
}
|
||||
const finalDocumentIdentity = await readDocumentIdentity();
|
||||
if (finalDocumentIdentity !== initialDocumentIdentity) {
|
||||
throw new Error(
|
||||
"Frame changed while its browser snapshot was being captured; retry.",
|
||||
);
|
||||
}
|
||||
};
|
||||
if (plan.format === "ai") {
|
||||
const roleSnapshotArgs = {
|
||||
cdpUrl: profileCtx.profile.cdpUrl,
|
||||
@@ -719,6 +816,7 @@ export function registerBrowserAgentSnapshotRoutes(
|
||||
compact: plan.compact ?? undefined,
|
||||
maxDepth: plan.depth ?? undefined,
|
||||
},
|
||||
delta: deltaState.delta,
|
||||
};
|
||||
|
||||
const cdpRoleSnapshot = async () => {
|
||||
@@ -738,6 +836,7 @@ export function registerBrowserAgentSnapshotRoutes(
|
||||
compact: plan.compact ?? undefined,
|
||||
maxDepth: plan.depth ?? undefined,
|
||||
},
|
||||
delta: deltaState.delta,
|
||||
});
|
||||
};
|
||||
|
||||
@@ -764,6 +863,7 @@ export function registerBrowserAgentSnapshotRoutes(
|
||||
...(typeof plan.resolvedMaxChars === "number"
|
||||
? { maxChars: plan.resolvedMaxChars }
|
||||
: {}),
|
||||
delta: deltaState.delta,
|
||||
})
|
||||
: await cdpRoleSnapshot();
|
||||
if (!snap) {
|
||||
@@ -801,6 +901,8 @@ export function registerBrowserAgentSnapshotRoutes(
|
||||
DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
|
||||
);
|
||||
const imageType = normalized.contentType?.includes("jpeg") ? "jpeg" : "png";
|
||||
await assertDocumentIdentityUnchanged();
|
||||
deltaState.record(snap.refs ?? {});
|
||||
return res.json({
|
||||
ok: true,
|
||||
format: plan.format,
|
||||
@@ -819,6 +921,8 @@ export function registerBrowserAgentSnapshotRoutes(
|
||||
});
|
||||
}
|
||||
|
||||
await assertDocumentIdentityUnchanged();
|
||||
deltaState.record(snap.refs ?? {});
|
||||
return res.json({
|
||||
ok: true,
|
||||
format: plan.format,
|
||||
|
||||
@@ -17,6 +17,7 @@ import {
|
||||
import { getBrowserProfileCapabilities } from "../profile-capabilities.js";
|
||||
import type { BrowserRouteContext, ProfileContext } from "../server-context.js";
|
||||
import { isProfileRestartRequiredError } from "../server-context.lifecycle.js";
|
||||
import { clearSnapshotKeysForTab } from "../snapshot-delta-cache.js";
|
||||
import { resolveTargetIdFromTabs } from "../target-id.js";
|
||||
import { browserNavigationPolicyForProfile, resolveProfileContext } from "./agent.shared.js";
|
||||
import { readRouteNonNegativeInteger } from "./route-numeric.js";
|
||||
@@ -321,6 +322,7 @@ export function registerBrowserTabRoutes(app: BrowserRouteRegistrar, ctx: Browse
|
||||
targetId,
|
||||
mutate: async (profileCtx, id) => {
|
||||
await profileCtx.closeTab(id, targetIdMode === "raw" ? { exactTargetId: true } : undefined);
|
||||
clearSnapshotKeysForTab(ctx, profileCtx.profile.name, id);
|
||||
},
|
||||
});
|
||||
});
|
||||
@@ -397,6 +399,7 @@ export function registerBrowserTabRoutes(app: BrowserRouteRegistrar, ctx: Browse
|
||||
throw new BrowserTabNotFoundError();
|
||||
}
|
||||
await profileCtx.closeTab(target.targetId, { exactTargetId: true });
|
||||
clearSnapshotKeysForTab(ctx, profileCtx.profile.name, target.targetId);
|
||||
return { ok: true, targetId: target.targetId };
|
||||
}
|
||||
|
||||
|
||||
@@ -107,6 +107,7 @@ const cdpMocks = vi.hoisted(() => ({
|
||||
createTargetViaCdp: vi.fn<() => Promise<{ targetId: string }>>(async () => {
|
||||
throw new Error("cdp disabled");
|
||||
}),
|
||||
getMainFrameDocumentIdentityViaCdp: vi.fn(async () => "cdp:test-document"),
|
||||
snapshotAria: vi.fn(async () => ({
|
||||
nodes: [{ ref: "1", role: "link", name: "x", depth: 0 }],
|
||||
})),
|
||||
@@ -120,11 +121,13 @@ const cdpMocks = vi.hoisted(() => ({
|
||||
/** Returns mocked CDP functions used by Browser control-server tests. */
|
||||
export function getCdpMocks(): {
|
||||
createTargetViaCdp: MockFn;
|
||||
getMainFrameDocumentIdentityViaCdp: MockFn;
|
||||
snapshotAria: MockFn;
|
||||
snapshotRoleViaCdp: MockFn;
|
||||
} {
|
||||
return cdpMocks as unknown as {
|
||||
createTargetViaCdp: MockFn;
|
||||
getMainFrameDocumentIdentityViaCdp: MockFn;
|
||||
snapshotAria: MockFn;
|
||||
snapshotRoleViaCdp: MockFn;
|
||||
};
|
||||
@@ -211,6 +214,7 @@ const pwMocks = vi.hoisted(() => {
|
||||
getObservedBrowserStateViaPlaywright: vi.fn(async () => ({
|
||||
dialogs: { pending: [], recent: [] },
|
||||
})),
|
||||
getMainFrameDocumentIdentityViaPlaywright: vi.fn(async () => "pw:test-document"),
|
||||
getPageErrorsViaPlaywright: vi.fn(async () => ({ errors: [] })),
|
||||
highlightViaPlaywright: vi.fn(async (_opts?: unknown) => {}),
|
||||
hoverViaPlaywright: vi.fn(async (_opts?: unknown) => {}),
|
||||
@@ -533,6 +537,7 @@ vi.mock("./chrome.js", () => ({
|
||||
|
||||
vi.mock("./cdp.js", () => ({
|
||||
createTargetViaCdp: cdpMocks.createTargetViaCdp,
|
||||
getMainFrameDocumentIdentityViaCdp: cdpMocks.getMainFrameDocumentIdentityViaCdp,
|
||||
normalizeCdpWsUrl: vi.fn((wsUrl: string) => wsUrl),
|
||||
snapshotAria: cdpMocks.snapshotAria,
|
||||
snapshotRoleViaCdp: cdpMocks.snapshotRoleViaCdp,
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { finalizeRoleSnapshot } from "./pw-role-snapshot.js";
|
||||
import type { BrowserRouteContext } from "./server-context.types.js";
|
||||
import {
|
||||
getPreviousSnapshotKeys,
|
||||
recordSnapshotKeys,
|
||||
type SnapshotDeltaFamily,
|
||||
} from "./snapshot-delta-cache.js";
|
||||
|
||||
const family: SnapshotDeltaFamily = { identity: "role", interactive: true };
|
||||
|
||||
function createContext(): BrowserRouteContext {
|
||||
return {} as BrowserRouteContext;
|
||||
}
|
||||
|
||||
function cacheScope(documentIdentity: string) {
|
||||
return { profile: "openclaw", targetId: "tab-1", documentIdentity, family };
|
||||
}
|
||||
|
||||
describe("snapshot delta cache", () => {
|
||||
it("starts an unmarked baseline after a same-URL reload", () => {
|
||||
const ctx = createContext();
|
||||
recordSnapshotKeys(ctx, {
|
||||
...cacheScope("pw:document-1"),
|
||||
refs: { e1: { role: "button", name: "Save" } },
|
||||
});
|
||||
|
||||
const previousKeys = getPreviousSnapshotKeys(ctx, cacheScope("pw:document-2"));
|
||||
const snapshot = finalizeRoleSnapshot({
|
||||
snapshot: '- button "Continue" [ref=e1]',
|
||||
refs: { e1: { role: "button", name: "Continue" } },
|
||||
delta: { mode: "role", previousKeys },
|
||||
});
|
||||
|
||||
expect(previousKeys).toBeUndefined();
|
||||
expect(snapshot.snapshot).toBe('- button "Continue" [ref=e1]');
|
||||
expect(snapshot.newElements).toBeUndefined();
|
||||
});
|
||||
|
||||
it("marks elements added in the same document", () => {
|
||||
const ctx = createContext();
|
||||
const scope = cacheScope("pw:document-1");
|
||||
recordSnapshotKeys(ctx, {
|
||||
...scope,
|
||||
refs: { e1: { role: "button", name: "Save" } },
|
||||
});
|
||||
|
||||
const snapshot = finalizeRoleSnapshot({
|
||||
snapshot: ['- button "Save" [ref=e7]', '- alert "Required" [ref=e8]'].join("\n"),
|
||||
refs: {
|
||||
e7: { role: "button", name: "Save" },
|
||||
e8: { role: "alert", name: "Required" },
|
||||
},
|
||||
delta: { mode: "role", previousKeys: getPreviousSnapshotKeys(ctx, scope) },
|
||||
});
|
||||
|
||||
expect(snapshot.snapshot).toContain('- alert "Required" [ref=e8] [new]');
|
||||
expect(snapshot.newElements).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,120 @@
|
||||
import {
|
||||
getRoleSnapshotIdentityKeys,
|
||||
type RoleRefMap,
|
||||
type RoleSnapshotIdentityMode,
|
||||
} from "./pw-role-snapshot.js";
|
||||
import type { BrowserRouteContext } from "./server-context.types.js";
|
||||
|
||||
/**
|
||||
* Process-local snapshot delta state owned by one Browser control-server context.
|
||||
* Each tab/options family keeps one bounded previous-key slot; restart or tab close drops it.
|
||||
*/
|
||||
const SNAPSHOT_DELTA_CACHE_MAX_ENTRIES = 32;
|
||||
|
||||
export type SnapshotDeltaFamily = {
|
||||
identity: RoleSnapshotIdentityMode;
|
||||
interactive?: boolean;
|
||||
compact?: boolean;
|
||||
depth?: number;
|
||||
selector?: string;
|
||||
frame?: string;
|
||||
urls?: boolean;
|
||||
maxChars?: number;
|
||||
};
|
||||
|
||||
type SnapshotDeltaEntry = {
|
||||
profile: string;
|
||||
targetId: string;
|
||||
documentIdentity: string;
|
||||
keys: Set<string>;
|
||||
};
|
||||
|
||||
const cacheByContext = new WeakMap<BrowserRouteContext, Map<string, SnapshotDeltaEntry>>();
|
||||
|
||||
function getCache(ctx: BrowserRouteContext): Map<string, SnapshotDeltaEntry> {
|
||||
const existing = cacheByContext.get(ctx);
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
const cache = new Map<string, SnapshotDeltaEntry>();
|
||||
cacheByContext.set(ctx, cache);
|
||||
return cache;
|
||||
}
|
||||
|
||||
function cacheKey(params: {
|
||||
profile: string;
|
||||
targetId: string;
|
||||
family: SnapshotDeltaFamily;
|
||||
}): string {
|
||||
return JSON.stringify([params.profile, params.targetId, params.family]);
|
||||
}
|
||||
|
||||
export function getPreviousSnapshotKeys(
|
||||
ctx: BrowserRouteContext,
|
||||
params: {
|
||||
profile: string;
|
||||
targetId: string;
|
||||
documentIdentity: string;
|
||||
family: SnapshotDeltaFamily;
|
||||
},
|
||||
): ReadonlySet<string> | undefined {
|
||||
const cache = getCache(ctx);
|
||||
const key = cacheKey(params);
|
||||
const entry = cache.get(key);
|
||||
if (!entry) {
|
||||
return undefined;
|
||||
}
|
||||
// Delta markers are same-document only. Navigation resets the baseline so a
|
||||
// replacement document is not reported as a tree full of newly appeared elements.
|
||||
if (entry.documentIdentity !== params.documentIdentity) {
|
||||
cache.delete(key);
|
||||
return undefined;
|
||||
}
|
||||
cache.delete(key);
|
||||
cache.set(key, entry);
|
||||
return entry.keys;
|
||||
}
|
||||
|
||||
export function recordSnapshotKeys(
|
||||
ctx: BrowserRouteContext,
|
||||
params: {
|
||||
profile: string;
|
||||
targetId: string;
|
||||
documentIdentity: string;
|
||||
family: SnapshotDeltaFamily;
|
||||
refs: RoleRefMap;
|
||||
},
|
||||
): void {
|
||||
const cache = getCache(ctx);
|
||||
const key = cacheKey(params);
|
||||
cache.delete(key);
|
||||
cache.set(key, {
|
||||
profile: params.profile,
|
||||
targetId: params.targetId,
|
||||
documentIdentity: params.documentIdentity,
|
||||
keys: getRoleSnapshotIdentityKeys(params.refs, params.family.identity),
|
||||
});
|
||||
while (cache.size > SNAPSHOT_DELTA_CACHE_MAX_ENTRIES) {
|
||||
const oldest = cache.keys().next().value as string | undefined;
|
||||
if (oldest === undefined) {
|
||||
break;
|
||||
}
|
||||
cache.delete(oldest);
|
||||
}
|
||||
}
|
||||
|
||||
export function clearSnapshotKeysForTab(
|
||||
ctx: BrowserRouteContext,
|
||||
profile: string,
|
||||
targetId: string,
|
||||
): void {
|
||||
const cache = cacheByContext.get(ctx);
|
||||
if (!cache) {
|
||||
return;
|
||||
}
|
||||
for (const [key, entry] of cache) {
|
||||
if (entry.profile === profile && entry.targetId === targetId) {
|
||||
cache.delete(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user