From 4dd8a2be762763a68936aff998e6a8efeeb418c2 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 25 Jul 2026 10:58:03 -0700 Subject: [PATCH] improve(browser): batch navigation guard, snapshot delta markers, tool output schema, code-mode recipe (#113749) * feat(browser): guard batches and annotate snapshot deltas * docs(browser): document reliable automation loops * test(browser): align batch guard mocks * fix(browser): reset snapshot deltas after navigation * fix(browser): detect document navigation without URL equality * fix(browser): require stable identity for snapshot deltas * fix(browser): keep snapshot delta helper private --- docs/tools/browser-control.md | 11 + docs/tools/browser.md | 5 + .../skills/browser-automation/SKILL.md | 38 ++++ .../browser/src/browser-tool-description.ts | 1 + .../browser/src/browser-tool.actions.ts | 39 +++- .../browser/src/browser-tool.runtime.ts | 2 +- extensions/browser/src/browser-tool.schema.ts | 75 ++++++ extensions/browser/src/browser-tool.test.ts | 56 ++++- extensions/browser/src/browser-tool.ts | 2 + .../src/browser/cdp-page-session.test.ts | 11 +- .../browser/src/browser/cdp-page-session.ts | 12 + extensions/browser/src/browser/cdp.ts | 18 +- .../src/browser/client-actions-core.ts | 5 +- .../src/browser/client-actions-types.ts | 16 ++ extensions/browser/src/browser/client.ts | 1 + extensions/browser/src/browser/pw-ai.ts | 2 + .../src/browser/pw-role-snapshot.test.ts | 70 ++++++ .../browser/src/browser/pw-role-snapshot.ts | 78 ++++++- .../browser/src/browser/pw-session-actions.ts | 14 +- .../src/browser/pw-session.page-cdp.test.ts | 20 ++ .../src/browser/pw-session.page-cdp.ts | 19 ++ extensions/browser/src/browser/pw-session.ts | 1 + .../pw-tools-core.interactions.batch.test.ts | 215 ++++++++++++++++-- .../src/browser/pw-tools-core.interactions.ts | 123 +++++++--- .../src/browser/pw-tools-core.snapshot.ts | 17 +- .../browser/src/browser/routes/agent.act.ts | 11 +- .../routes/agent.existing-session.test.ts | 29 +++ .../agent.snapshot.local-managed.test.ts | 56 ++++- .../routes/agent.snapshot.timeout.test.ts | 2 + .../src/browser/routes/agent.snapshot.ts | 128 ++++++++++- extensions/browser/src/browser/routes/tabs.ts | 3 + .../server.control-server.test-harness.ts | 5 + .../src/browser/snapshot-delta-cache.test.ts | 60 +++++ .../src/browser/snapshot-delta-cache.ts | 120 ++++++++++ 34 files changed, 1197 insertions(+), 68 deletions(-) create mode 100644 extensions/browser/src/browser/snapshot-delta-cache.test.ts create mode 100644 extensions/browser/src/browser/snapshot-delta-cache.ts diff --git a/docs/tools/browser-control.md b/docs/tools/browser-control.md index 72ccf35255db..5c3eac14b30f 100644 --- a/docs/tools/browser-control.md +++ b/docs/tools/browser-control.md @@ -328,6 +328,13 @@ OpenClaw supports two "snapshot" styles: - If Playwright is unavailable, ARIA snapshots can still be useful for inspection, but refs may not be actionable. Re-snapshot with `--format ai` or `--interactive` when you need action refs. +- When the driver exposes stable document identity, consecutive AI and role + snapshots for the same profile, tab, document, and option family append + `[new]` to ref-bearing lines absent from the previous snapshot. Navigation + starts a fresh unmarked baseline; existing-session snapshots omit deltas. + The first snapshot establishes the baseline without markers; later responses + also expose `newElements`, and add a count footer when the value is nonzero. + Structured `--format aria` snapshots with `axN` refs do not use delta markers. - Docker proof for the raw-CDP fallback path: `pnpm test:docker:browser-cdp-snapshot` starts Chromium with CDP, runs `browser doctor --deep`, and verifies role snapshots include link URLs, cursor-promoted clickables, and iframe metadata. @@ -335,6 +342,10 @@ OpenClaw supports two "snapshot" styles: Ref behavior: - Refs are **not stable across navigations**; if something fails, re-run `snapshot` and use a fresh ref. +- A batch stops after a committed main-frame navigation—including a same-URL + reload—or after the page closes. Its `aborted` summary reports the action + number and skipped count; take a fresh snapshot before issuing dependent + actions, or use separate act calls when navigation is expected. - `/act` returns the current raw `targetId` after action-triggered replacement when it can prove the replacement tab. Keep using stable tab ids/labels for follow-up commands. diff --git a/docs/tools/browser.md b/docs/tools/browser.md index 2a6486906a42..655fc589243d 100644 --- a/docs/tools/browser.md +++ b/docs/tools/browser.md @@ -197,6 +197,11 @@ extraction mode when a caller does not pass an explicit `snapshotFormat` or `mode`; see [Browser control API](/tools/browser-control) for per-call snapshot options. +On drivers with stable document identity, repeated AI or role snapshots of the +same tab, document, and option family mark newly appeared ref-bearing elements +with `[new]`. The first snapshot—and the first snapshot after navigation—sets +an unmarked baseline. Existing-session snapshots omit deltas. + ### Tab cleanup ownership Session tab cleanup applies only to tabs created by the OpenClaw browser tool diff --git a/extensions/browser/skills/browser-automation/SKILL.md b/extensions/browser/skills/browser-automation/SKILL.md index 71e68dc197d1..bdb765a480e4 100644 --- a/extensions/browser/skills/browser-automation/SKILL.md +++ b/extensions/browser/skills/browser-automation/SKILL.md @@ -43,6 +43,44 @@ Use this skill when you need the `browser` tool for anything beyond a single pag - Target id: nested actions share the request's tab; an explicit nested `targetId` that resolves to a different tab is rejected with `ACT_TARGET_ID_MISMATCH`. - Response: `{ "results": [{ "ok": true } | { "ok": false, "error": "..." }, ...] }` in order; with default `stopOnError` the array ends at the first failure. Any failed entry exits nonzero; use `--json` to preserve the full response in scripts. +## Code Mode Loop + +When `tools.codeMode` is enabled, call the Browser tool from exec cells: + +```javascript +const browserTool = "openclaw:browser:browser"; +let previousSnapshot = ""; +const callBrowser = async (input) => await tools.call(browserTool, input); +``` + +Keep the same labeled tab through the loop, and alternate reads with actions: + +```javascript +const snapshotCall = await callBrowser({ + action: "snapshot", + targetId: "task", + refs: "aria", + interactive: true, +}); +const details = snapshotCall?.result?.details ?? {}; +const snapshot = (snapshotCall?.result?.content ?? []).map((block) => block?.text ?? "").join("\n"); +const relevant = snapshot + .split("\n") + .filter((line) => /submit|dialog|error|\[new\]/i.test(line)) + .slice(0, 12); +const changed = snapshot !== previousSnapshot; +previousSnapshot = snapshot; +return { targetId: details.targetId, url: details.url, relevant, changed }; +``` + +- Request interactive-only snapshots and filter them in code before returning. +- Return only the handful of relevant elements; never return the full tree. +- Keep `previousSnapshot` between cells when a local diff helps explain a change. +- Interleave each act with a URL or tabs check before the next dependent act. +- If a batch returns `aborted`, take a fresh snapshot before continuing. +- If `[new]` markers appear, inspect those elements first, then update the saved snapshot. +- Use separate act calls when navigation is expected between steps. + ## Tab Hygiene Before creating a tab for a named task, list tabs and reuse an existing matching label or URL when it is still usable. diff --git a/extensions/browser/src/browser-tool-description.ts b/extensions/browser/src/browser-tool-description.ts index f58f9520bf4c..0b6996501141 100644 --- a/extensions/browser/src/browser-tool-description.ts +++ b/extensions/browser/src/browser-tool-description.ts @@ -13,6 +13,7 @@ export function describeBrowserTool(opts: { "When using refs from snapshot (e.g. e12), keep the same tab: prefer passing targetId from the snapshot response into subsequent actions (act/click/type/etc). For tab operations, targetId also accepts tabId handles (t1) and labels from action=tabs.", "For multi-step browser work, login checks, stale refs, duplicate tabs, or Google Meet flows, use the bundled browser-automation skill when it is available.", 'For stable, self-resolving refs across calls, use snapshot with refs="aria" (Playwright aria-ref ids). Default refs="role" are role+name-based.', + "Repeated compatible snapshots with stable document identity mark newly appeared ref-bearing elements with [new].", "Use snapshot+act for UI automation. Avoid act:wait by default; use only in exceptional cases when no reliable UI state exists.", "For file chooser uploads, pass the trigger ref with paths in the same upload call when available; use paths-only arming only when a later trigger is intentional. Use inputRef or element to set a file input directly.", `target selects browser location (sandbox|host|node). Default: ${opts.targetDefault}.`, diff --git a/extensions/browser/src/browser-tool.actions.ts b/extensions/browser/src/browser-tool.actions.ts index 7d4609dbeb17..8360026eb149 100644 --- a/extensions/browser/src/browser-tool.actions.ts +++ b/extensions/browser/src/browser-tool.actions.ts @@ -459,6 +459,7 @@ export async function executeSnapshotAction(params: { targetId: snapshot.targetId, url: snapshot.url, truncated: snapshot.truncated, + newElements: snapshot.newElements, stats: snapshot.stats, refs: snapshot.refs ? Object.keys(snapshot.refs).length : undefined, labels: snapshot.labels, @@ -646,7 +647,7 @@ export async function executeActAction(params: { readStringValue((result as { targetId?: unknown }).targetId) ?? readStringValue(effectiveRequest.targetId), ); - return jsonResult(result); + return formatActToolResult(result); } catch (err) { if (isChromeStaleTargetError(profile, err)) { const tabs = proxyRequest @@ -688,7 +689,7 @@ export async function executeActAction(params: { readStringValue((retryResult as { targetId?: unknown }).targetId) ?? readStringValue(retryRequest.targetId), ); - return jsonResult(retryResult); + return formatActToolResult(retryResult); } if (!tabs.length) { throw new Error( @@ -704,3 +705,37 @@ export async function executeActAction(params: { throw err; } } + +function formatActToolResult(result: unknown): AgentToolResult { + const formatted = jsonResult(result); + if (!result || typeof result !== "object") { + return formatted; + } + const aborted = (result as { aborted?: unknown }).aborted; + if (!aborted || typeof aborted !== "object") { + return formatted; + } + const summary = aborted as { + reason?: unknown; + afterAction?: unknown; + url?: unknown; + skipped?: unknown; + }; + if ( + (summary.reason !== "navigation" && summary.reason !== "closed") || + typeof summary.afterAction !== "number" || + typeof summary.url !== "string" || + typeof summary.skipped !== "number" + ) { + return formatted; + } + const reason = + summary.reason === "navigation" + ? `the page navigated to ${summary.url}` + : "the page or browser context closed"; + const note = `Batch aborted after action ${summary.afterAction} because ${reason}; ${summary.skipped} remaining action(s) skipped. Take a new snapshot before continuing.`; + return { + ...formatted, + content: [...formatted.content, { type: "text", text: note }], + }; +} diff --git a/extensions/browser/src/browser-tool.runtime.ts b/extensions/browser/src/browser-tool.runtime.ts index a3e31da77201..051b9e2dae69 100644 --- a/extensions/browser/src/browser-tool.runtime.ts +++ b/extensions/browser/src/browser-tool.runtime.ts @@ -33,7 +33,7 @@ export { normalizeOptionalString, readStringValue, } from "openclaw/plugin-sdk/string-coerce-runtime"; -export { BrowserToolSchema } from "./browser-tool.schema.js"; +export { BrowserToolOutputSchema, BrowserToolSchema } from "./browser-tool.schema.js"; export { browserAct, browserArmDialog, diff --git a/extensions/browser/src/browser-tool.schema.ts b/extensions/browser/src/browser-tool.schema.ts index 200fd4fff725..cdf0fe67237e 100644 --- a/extensions/browser/src/browser-tool.schema.ts +++ b/extensions/browser/src/browser-tool.schema.ts @@ -177,3 +177,78 @@ export const BrowserToolSchema = Type.Object({ fn: Type.Optional(Type.String()), request: Type.Optional(BrowserActSchema), }); + +const BrowserSnapshotStatsSchema = Type.Object( + { + lines: Type.Number(), + chars: Type.Number(), + refs: Type.Number(), + interactive: Type.Number(), + }, + { additionalProperties: false }, +); + +const BrowserBatchAbortSchema = Type.Object( + { + reason: stringEnum(["navigation", "closed"] as const), + afterAction: Type.Number(), + url: Type.String(), + skipped: Type.Number(), + }, + { additionalProperties: false }, +); + +/** Common structured result fields returned across Browser tool actions. */ +export const BrowserToolOutputSchema = Type.Object( + { + ok: Type.Optional(Type.Boolean()), + targetId: Type.Optional(Type.String()), + url: Type.Optional(Type.String()), + format: Type.Optional(stringEnum(BROWSER_SNAPSHOT_FORMATS)), + snapshot: Type.Optional(Type.String()), + refs: Type.Optional(Type.Union([Type.Number(), Type.Record(Type.String(), Type.Unknown())])), + stats: Type.Optional(BrowserSnapshotStatsSchema), + truncated: Type.Optional(Type.Boolean()), + newElements: Type.Optional(Type.Number()), + tabs: Type.Optional( + Type.Array( + Type.Object( + { + suggestedTargetId: Type.Optional(Type.String()), + tabId: Type.Optional(Type.String()), + label: Type.Optional(Type.String()), + targetId: Type.Optional(Type.String()), + title: Type.Optional(Type.String()), + url: Type.Optional(Type.String()), + type: Type.Optional(Type.String()), + }, + { additionalProperties: true }, + ), + ), + ), + tabCount: Type.Optional(Type.Number()), + results: Type.Optional( + Type.Array( + Type.Object( + { + ok: Type.Boolean(), + error: Type.Optional(Type.String()), + navigated: Type.Optional(Type.Literal(true)), + url: Type.Optional(Type.String()), + }, + { additionalProperties: false }, + ), + ), + ), + aborted: Type.Optional(BrowserBatchAbortSchema), + enabled: Type.Optional(Type.Boolean()), + running: Type.Optional(Type.Boolean()), + profile: Type.Optional(Type.String()), + driver: Type.Optional(Type.String()), + transport: Type.Optional(Type.String()), + pid: Type.Optional(Type.Union([Type.Number(), Type.Null()])), + cdpPort: Type.Optional(Type.Union([Type.Number(), Type.Null()])), + cdpUrl: Type.Optional(Type.Union([Type.String(), Type.Null()])), + }, + { additionalProperties: true }, +); diff --git a/extensions/browser/src/browser-tool.test.ts b/extensions/browser/src/browser-tool.test.ts index d0c7763c3bd2..ac58d03088e4 100644 --- a/extensions/browser/src/browser-tool.test.ts +++ b/extensions/browser/src/browser-tool.test.ts @@ -1,4 +1,5 @@ // Browser tests cover browser tool plugin behavior. +import { Value } from "typebox/value"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; const browserClientMocks = vi.hoisted(() => ({ @@ -55,7 +56,7 @@ const browserClientMocks = vi.hoisted(() => ({ vi.mock("./browser/client.js", () => browserClientMocks); const browserActionsMocks = vi.hoisted(() => ({ - browserAct: vi.fn(async () => ({ ok: true })), + browserAct: vi.fn(async (): Promise> => ({ ok: true })), browserArmDialog: vi.fn(async () => ({ ok: true })), browserArmFileChooser: vi.fn(async () => ({ ok: true })), browserConsoleMessages: vi.fn(async () => ({ @@ -213,7 +214,10 @@ vi.mock("./sdk-setup-tools.js", async () => { }; }); -vi.mock("./browser-tool.runtime.js", () => { +vi.mock("./browser-tool.runtime.js", async () => { + const { BrowserToolOutputSchema } = await vi.importActual< + typeof import("./browser-tool.schema.js") + >("./browser-tool.schema.js"); const readStringValue = (value: unknown) => (typeof value === "string" ? value : undefined); const readStringParam = ( params: Record, @@ -233,6 +237,7 @@ vi.mock("./browser-tool.runtime.js", () => { return { DEFAULT_AI_SNAPSHOT_MAX_CHARS: 40_000, DEFAULT_UPLOAD_DIR: "/tmp/openclaw-browser-uploads", + BrowserToolOutputSchema, BrowserToolSchema: {}, ...browserActionsMocks, ...browserClientMocks, @@ -487,6 +492,20 @@ function lastNodeInvokeCall(): ReturnType { return nodeInvokeCall(-1); } +describe("browser tool output schema", () => { + it("accepts snapshot details", async () => { + const tool = createBrowserTool(); + const result = await tool.execute?.("call-1", { + action: "snapshot", + target: "host", + snapshotFormat: "ai", + }); + + expect(tool.outputSchema).toBeDefined(); + expect(Value.Check(tool.outputSchema!, result?.details)).toBe(true); + }); +}); + describe("browser tool description", () => { it("warns agents about existing-session act timeout limits", () => { const tool = createBrowserTool(); @@ -2257,6 +2276,39 @@ describe("browser tool url alias support", () => { describe("browser tool act compatibility", () => { registerBrowserToolAfterEachReset(); + it("adds a clear note when a batch aborts after navigation", async () => { + browserActionsMocks.browserAct.mockResolvedValueOnce({ + ok: true, + results: [{ ok: true, navigated: true, url: "https://example.com/next" }], + aborted: { + reason: "navigation", + afterAction: 1, + url: "https://example.com/next", + skipped: 2, + }, + }); + const tool = createBrowserTool(); + + const result = await tool.execute?.("call-1", { + action: "act", + request: { + kind: "batch", + actions: [ + { kind: "click", ref: "1" }, + { kind: "click", ref: "2" }, + ], + }, + }); + + expect(result?.details).toMatchObject({ aborted: { reason: "navigation", skipped: 2 } }); + expect(result?.content.at(-1)).toMatchObject({ + type: "text", + text: expect.stringContaining( + "Batch aborted after action 1 because the page navigated to https://example.com/next", + ), + }); + }); + it("accepts flattened act params for backward compatibility", async () => { const tool = createBrowserTool(); await tool.execute?.("call-1", { diff --git a/extensions/browser/src/browser-tool.ts b/extensions/browser/src/browser-tool.ts index 4f9037dcbef1..1686a923d253 100644 --- a/extensions/browser/src/browser-tool.ts +++ b/extensions/browser/src/browser-tool.ts @@ -21,6 +21,7 @@ import { import { type AnyAgentTool, type NodeListNode, + BrowserToolOutputSchema, BrowserToolSchema, browserAct, browserArmDialog, @@ -418,6 +419,7 @@ export function createBrowserTool(opts?: { name: "browser", description: describeBrowserTool({ targetDefault, hostHint }), parameters: BrowserToolSchema, + outputSchema: BrowserToolOutputSchema, execute: async (_toolCallId, args) => { const bindingResult = opts?.runToolBinding === undefined diff --git a/extensions/browser/src/browser/cdp-page-session.test.ts b/extensions/browser/src/browser/cdp-page-session.test.ts index ffe9957796b6..3ef8a64eecba 100644 --- a/extensions/browser/src/browser/cdp-page-session.test.ts +++ b/extensions/browser/src/browser/cdp-page-session.test.ts @@ -1,9 +1,18 @@ // Browser tests cover CDP committed page-session URL observation. import { describe, expect, it } from "vitest"; -import { prepareCdpTargetSession } from "./cdp-page-session.js"; +import { prepareCdpTargetSession, readCdpMainFrameDocumentIdentity } from "./cdp-page-session.js"; import type { CdpSendFn } from "./cdp.helpers.js"; describe("prepareCdpTargetSession", () => { + it("reads the main-frame loader identity", async () => { + const send: CdpSendFn = async (method) => + method === "Page.getFrameTree" + ? { frameTree: { frame: { loaderId: "LOADER_SAME_URL" } } } + : {}; + + await expect(readCdpMainFrameDocumentIdentity(send)).resolves.toBe("cdp:LOADER_SAME_URL"); + }); + it("ignores Chrome's transient colon URL while navigation is settling", async () => { let frameReadCount = 0; const send: CdpSendFn = async (method) => { diff --git a/extensions/browser/src/browser/cdp-page-session.ts b/extensions/browser/src/browser/cdp-page-session.ts index c495e18e824b..0d2348e53350 100644 --- a/extensions/browser/src/browser/cdp-page-session.ts +++ b/extensions/browser/src/browser/cdp-page-session.ts @@ -42,6 +42,18 @@ function readCommittedFrameUrl( return url ? `${url}${fragment}` : undefined; } +/** Read the browser-owned loader identity for the committed main-frame document. */ +export async function readCdpMainFrameDocumentIdentity( + send: CdpSendFn, + sessionId?: string, +): Promise { + const frameTree = (await send("Page.getFrameTree", undefined, sessionId).catch( + () => null, + )) as CdpFrameTreeResult | null; + const loaderId = frameTree?.frameTree?.frame?.loaderId; + return typeof loaderId === "string" && loaderId.trim() ? `cdp:${loaderId.trim()}` : undefined; +} + async function waitForCdpNavigationResult( send: CdpSendFn, sessionId: string | undefined, diff --git a/extensions/browser/src/browser/cdp.ts b/extensions/browser/src/browser/cdp.ts index 5d5db7969d70..0892302ad6b4 100644 --- a/extensions/browser/src/browser/cdp.ts +++ b/extensions/browser/src/browser/cdp.ts @@ -11,6 +11,7 @@ import type { SsrFPolicy } from "../infra/net/ssrf.js"; import { prepareCdpPageSession, prepareCdpTargetSession, + readCdpMainFrameDocumentIdentity, type CdpActionTimeouts, } from "./cdp-page-session.js"; import { @@ -26,12 +27,24 @@ import { withCdpSocket, } from "./cdp.helpers.js"; import { assertBrowserNavigationAllowed, withBrowserNavigationPolicy } from "./navigation-guard.js"; -import { finalizeRoleSnapshot } from "./pw-role-snapshot.js"; +import { finalizeRoleSnapshot, type RoleSnapshotIdentityMode } from "./pw-role-snapshot.js"; import { CONTENT_ROLES, INTERACTIVE_ROLES, STRUCTURAL_ROLES } from "./snapshot-roles.js"; export { appendCdpPath } from "./cdp.helpers.js"; export { type CdpActionTimeouts, waitForCdpCommittedNavigationUrl } from "./cdp-page-session.js"; +/** Read the current main-frame loader identity from a page-level CDP target. */ +export async function getMainFrameDocumentIdentityViaCdp(opts: { + wsUrl: string; + timeoutMs?: number; +}): Promise { + return await withCdpSocket( + opts.wsUrl, + async (send) => await readCdpMainFrameDocumentIdentity(send), + { commandTimeoutMs: opts.timeoutMs ?? 5000 }, + ); +} + /** Normalize a reported CDP WebSocket URL against the configured CDP base URL. */ export function normalizeCdpWsUrl(wsUrl: string, cdpUrl: string): string { const ws = new URL(wsUrl); @@ -891,11 +904,13 @@ export async function snapshotRoleViaCdp(opts: { urls?: boolean; timeoutMs?: number; maxChars?: number; + delta?: { mode: RoleSnapshotIdentityMode; previousKeys?: ReadonlySet }; }): Promise<{ snapshot: string; truncated?: boolean; refs: Record; stats: { lines: number; chars: number; refs: number; interactive: number }; + newElements?: number; }> { return await withCdpSocket( opts.wsUrl, @@ -915,6 +930,7 @@ export async function snapshotRoleViaCdp(opts: { snapshot, refs: built.refs, maxChars: opts.maxChars, + delta: opts.delta, }); }, { commandTimeoutMs: opts.timeoutMs ?? 5000 }, diff --git a/extensions/browser/src/browser/client-actions-core.ts b/extensions/browser/src/browser/client-actions-core.ts index 7174ba8d623e..a7f1ab8cb1f5 100644 --- a/extensions/browser/src/browser/client-actions-core.ts +++ b/extensions/browser/src/browser/client-actions-core.ts @@ -17,6 +17,8 @@ import type { BrowserActionOk, BrowserActionPathResult, BrowserActionTabResult, + BrowserBatchAbort, + BrowserBatchActionResult, } from "./client-actions-types.js"; import { buildProfileQuery, withBaseUrl } from "./client-actions-url.js"; import type { BrowserActRequest } from "./client-actions.types.js"; @@ -34,7 +36,8 @@ type BrowserActResponse = { targetId: string; url?: string; result?: unknown; - results?: Array<{ ok: boolean; error?: string }>; + results?: BrowserBatchActionResult[]; + aborted?: BrowserBatchAbort; blockedByDialog?: boolean; browserState?: unknown; /** Download info when a click/batch/evaluate action triggers a browser download. */ diff --git a/extensions/browser/src/browser/client-actions-types.ts b/extensions/browser/src/browser/client-actions-types.ts index 3617c08309f8..54aa8c0a1ca3 100644 --- a/extensions/browser/src/browser/client-actions-types.ts +++ b/extensions/browser/src/browser/client-actions-types.ts @@ -7,6 +7,22 @@ import type { AnnotationItem } from "./screenshot-annotate.js"; /** Generic success result for action endpoints. */ export type BrowserActionOk = { ok: true }; +/** Per-action result returned by a browser batch. */ +export type BrowserBatchActionResult = { + ok: boolean; + error?: string; + navigated?: true; + url?: string; +}; + +/** Summary returned when a batch cannot safely continue on its original page. */ +export type BrowserBatchAbort = { + reason: "navigation" | "closed"; + afterAction: number; + url: string; + skipped: number; +}; + /** Success result carrying the affected tab and optional URL. */ export type BrowserActionTabResult = { ok: true; diff --git a/extensions/browser/src/browser/client.ts b/extensions/browser/src/browser/client.ts index 65bc38e8c5e7..496300f7cece 100644 --- a/extensions/browser/src/browser/client.ts +++ b/extensions/browser/src/browser/client.ts @@ -140,6 +140,7 @@ export type SnapshotResult = url: string; snapshot: string; truncated?: boolean; + newElements?: number; refs?: Record; stats?: { lines: number; diff --git a/extensions/browser/src/browser/pw-ai.ts b/extensions/browser/src/browser/pw-ai.ts index 8f33cc4351af..0d7c94ee10c0 100644 --- a/extensions/browser/src/browser/pw-ai.ts +++ b/extensions/browser/src/browser/pw-ai.ts @@ -9,6 +9,7 @@ import { forceDisconnectPlaywrightForTarget, getObservedBrowserStateForPage, getObservedBrowserStateViaPlaywright, + getMainFrameDocumentIdentityViaPlaywright, getPageForTargetId, isBrowserObservedDialogBlockedError, listPagesViaPlaywright, @@ -92,6 +93,7 @@ export const pwAi = { createObservedDialogAbortSignalForPage, getObservedBrowserStateForPage, getObservedBrowserStateViaPlaywright, + getMainFrameDocumentIdentityViaPlaywright, getPageForTargetId, isBrowserObservedDialogBlockedError, listPagesViaPlaywright, diff --git a/extensions/browser/src/browser/pw-role-snapshot.test.ts b/extensions/browser/src/browser/pw-role-snapshot.test.ts index c08d404ff731..9a272a1d721f 100644 --- a/extensions/browser/src/browser/pw-role-snapshot.test.ts +++ b/extensions/browser/src/browser/pw-role-snapshot.test.ts @@ -4,6 +4,7 @@ import { buildRoleSnapshotFromAiSnapshot, buildRoleSnapshotFromAriaSnapshot, finalizeRoleSnapshot, + getRoleSnapshotIdentityKeys, parseRoleRef, } from "./pw-role-snapshot.js"; @@ -131,6 +132,75 @@ describe("pw-role-snapshot", () => { expect(result.refs).toEqual({ e1: { role: "button" } }); }); + it("does not mark the first snapshot", () => { + const snapshot = '- button "Save" [ref=e1]'; + const refs = { e1: { role: "button", name: "Save" } }; + + const result = finalizeRoleSnapshot({ snapshot, refs, delta: { mode: "role" } }); + + expect(result.snapshot).toBe(snapshot); + expect(result.newElements).toBeUndefined(); + }); + + it("marks only new role identities and preserves ref extraction", () => { + const previousKeys = getRoleSnapshotIdentityKeys( + { e1: { role: "button", name: "Save" } }, + "role", + ); + const refs = { + e7: { role: "button", name: "Save" }, + e8: { role: "dialog", name: "Confirmation" }, + }; + const finalized = finalizeRoleSnapshot({ + snapshot: ['- button "Save" [ref=e7]', '- dialog "Confirmation" [ref=e8]'].join("\n"), + refs, + delta: { mode: "role", previousKeys }, + }); + + expect(finalized.snapshot).toBe( + [ + '- button "Save" [ref=e7]', + '- dialog "Confirmation" [ref=e8] [new]', + "1 new element(s) since last snapshot", + ].join("\n"), + ); + expect(finalized.newElements).toBe(1); + expect(finalized.refs).toEqual(refs); + }); + + it("uses preserved aria refs as AI snapshot identities", () => { + const finalized = finalizeRoleSnapshot({ + snapshot: ['- button "Save" [ref=7]', '- dialog "Confirmation" [ref=8]'].join("\n"), + refs: { + "7": { role: "button", name: "Save" }, + "8": { role: "dialog", name: "Confirmation" }, + }, + delta: { mode: "aria", previousKeys: new Set(["7"]) }, + }); + + expect(finalized.snapshot).toContain('- button "Save" [ref=7]\n'); + expect(finalized.snapshot).toContain('- dialog "Confirmation" [ref=8] [new]'); + expect(finalized.newElements).toBe(1); + }); + + it("annotates before truncation and keeps only complete annotated refs", () => { + const first = '- button "Visible" [ref=e1] [new]'; + const marker = "[...TRUNCATED - page too large]"; + const result = finalizeRoleSnapshot({ + snapshot: ['- button "Visible" [ref=e1]', '- dialog "Hidden" [ref=e2]'].join("\n"), + refs: { + e1: { role: "button", name: "Visible" }, + e2: { role: "dialog", name: "Hidden" }, + }, + maxChars: first.length + 2 + marker.length, + delta: { mode: "role", previousKeys: new Set() }, + }); + + expect(result.snapshot).toBe(`${first}\n\n${marker}`); + expect(result.refs).toEqual({ e1: { role: "button", name: "Visible" } }); + expect(result.newElements).toBe(1); + }); + it("treats sub-unit internal budgets as uncapped", () => { const snapshot = '- button "Visible" [ref=e1]'; const result = finalizeRoleSnapshot({ diff --git a/extensions/browser/src/browser/pw-role-snapshot.ts b/extensions/browser/src/browser/pw-role-snapshot.ts index 86485954ee2c..0bd5566b4662 100644 --- a/extensions/browser/src/browser/pw-role-snapshot.ts +++ b/extensions/browser/src/browser/pw-role-snapshot.ts @@ -18,6 +18,9 @@ type RoleRef = { /** Mapping from generated role refs to role/name metadata. */ export type RoleRefMap = Record; +/** Identity strategy used to compare consecutive ref-bearing snapshots. */ +export type RoleSnapshotIdentityMode = "role" | "aria"; + type RoleSnapshotStats = { lines: number; chars: number; @@ -57,6 +60,54 @@ function findSnapshotLineRef(line: string): string | undefined { return ROLE_SNAPSHOT_LINE_REF_RE.exec(line)?.[1]; } +/** Build the stable identity set used for per-tab snapshot deltas. */ +export function getRoleSnapshotIdentityKeys( + refs: Record, + mode: RoleSnapshotIdentityMode, +): Set { + // Duplicate role+name elements are identified positionally by nth, so insertion can mark a + // sibling duplicate. This is acceptable: they are actor-indistinguishable without DOM backing. + return new Set( + Object.entries(refs).map(([ref, value]) => + mode === "aria" ? ref : `${value.role}\0${value.name ?? ""}\0${value.nth ?? 0}`, + ), + ); +} + +/** Mark ref-bearing lines that were absent from the previous compatible snapshot. */ +function annotateRoleSnapshotDelta(params: { + snapshot: string; + refs: Record; + mode: RoleSnapshotIdentityMode; + previousKeys?: ReadonlySet; +}): { snapshot: string; keys: Set; newElements?: number } { + const keys = getRoleSnapshotIdentityKeys(params.refs, params.mode); + if (params.previousKeys === undefined) { + return { snapshot: params.snapshot, keys }; + } + const keyByRef = new Map( + Object.entries(params.refs).map(([ref, value]) => [ + ref, + params.mode === "aria" ? ref : `${value.role}\0${value.name ?? ""}\0${value.nth ?? 0}`, + ]), + ); + const markedKeys = new Set(); + const lines = params.snapshot.split("\n").map((line) => { + const ref = findSnapshotLineRef(line); + const key = ref ? keyByRef.get(ref) : undefined; + if (!key || params.previousKeys?.has(key)) { + return line; + } + markedKeys.add(key); + return `${line} [new]`; + }); + const newElements = markedKeys.size; + if (newElements > 0) { + lines.push(`${newElements} new element(s) since last snapshot`); + } + return { snapshot: lines.join("\n"), keys, newElements }; +} + function truncateRoleSnapshot(snapshot: string, maxChars: number): string { const marker = maxChars >= ROLE_SNAPSHOT_TRUNCATION_MARKER.length ? ROLE_SNAPSHOT_TRUNCATION_MARKER : "…"; @@ -72,23 +123,37 @@ function truncateRoleSnapshot(snapshot: string, maxChars: number): string { } /** Apply the final output budget, then keep only refs present on complete output lines. */ -export function finalizeRoleSnapshot(params: { +export function finalizeRoleSnapshot(params: { snapshot: string; refs: Record; maxChars?: number; + delta?: { + mode: RoleSnapshotIdentityMode; + previousKeys?: ReadonlySet; + }; }): { snapshot: string; truncated?: boolean; refs: Record; stats: RoleSnapshotStats; + newElements?: number; } { const normalizedMaxChars = typeof params.maxChars === "number" && Number.isFinite(params.maxChars) && params.maxChars > 0 ? Math.floor(params.maxChars) : undefined; const maxChars = normalizedMaxChars && normalizedMaxChars > 0 ? normalizedMaxChars : undefined; - const truncated = maxChars !== undefined && params.snapshot.length > maxChars; - const snapshot = truncated ? truncateRoleSnapshot(params.snapshot, maxChars) : params.snapshot; + const annotated = params.delta + ? annotateRoleSnapshotDelta({ + snapshot: params.snapshot, + refs: params.refs, + mode: params.delta.mode, + previousKeys: params.delta.previousKeys, + }) + : undefined; + const sourceSnapshot = annotated?.snapshot ?? params.snapshot; + const truncated = maxChars !== undefined && sourceSnapshot.length > maxChars; + const snapshot = truncated ? truncateRoleSnapshot(sourceSnapshot, maxChars) : sourceSnapshot; const visibleRefs = new Set( snapshot .split("\n") @@ -98,10 +163,17 @@ export function finalizeRoleSnapshot(params: { const refs = Object.fromEntries( Object.entries(params.refs).filter(([ref]) => visibleRefs.has(ref)), ) as Record; + const newElements = + params.delta?.previousKeys === undefined + ? undefined + : [...getRoleSnapshotIdentityKeys(refs, params.delta.mode)].filter( + (key) => !params.delta?.previousKeys?.has(key), + ).length; const result = { snapshot, refs, stats: getRoleSnapshotStats(snapshot, refs), + ...(newElements !== undefined ? { newElements } : {}), }; return truncated ? { ...result, truncated: true } : result; } diff --git a/extensions/browser/src/browser/pw-session-actions.ts b/extensions/browser/src/browser/pw-session-actions.ts index c2bb7c11677a..324dc34efcc6 100644 --- a/extensions/browser/src/browser/pw-session-actions.ts +++ b/extensions/browser/src/browser/pw-session-actions.ts @@ -52,7 +52,10 @@ import { getObservedBrowserStateForPage, normalizeCdpUrl, } from "./pw-session-state.js"; -import { BROWSER_REF_MARKER_ATTRIBUTE } from "./pw-session.page-cdp.js"; +import { + BROWSER_REF_MARKER_ATTRIBUTE, + readMainFrameDocumentIdentityForPage, +} from "./pw-session.page-cdp.js"; export async function getObservedBrowserStateViaPlaywright(opts: { cdpUrl: string; @@ -63,6 +66,15 @@ export async function getObservedBrowserStateViaPlaywright(opts: { return getObservedBrowserStateForPage(page); } +/** Resolve a page and read its current main-frame document identity. */ +export async function getMainFrameDocumentIdentityViaPlaywright(opts: { + cdpUrl: string; + targetId?: string; +}): Promise { + const page = await getPageForTargetId(opts); + return await readMainFrameDocumentIdentityForPage(page); +} + export function refLocator(page: Page, ref: string) { const normalized = ref.startsWith("@") ? ref.slice(1) diff --git a/extensions/browser/src/browser/pw-session.page-cdp.test.ts b/extensions/browser/src/browser/pw-session.page-cdp.test.ts index f20d8bf50ef0..01c99530a527 100644 --- a/extensions/browser/src/browser/pw-session.page-cdp.test.ts +++ b/extensions/browser/src/browser/pw-session.page-cdp.test.ts @@ -3,6 +3,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { BROWSER_REF_MARKER_ATTRIBUTE, markBackendDomRefsOnPage, + readMainFrameDocumentIdentityForPage, withPageScopedCdpClient, } from "./pw-session.page-cdp.js"; @@ -38,6 +39,25 @@ describe("pw-session page-scoped CDP client", () => { expect(sessionDetach).toHaveBeenCalledTimes(1); }); + it("reads the main-frame loader identity through the existing page session", async () => { + const sessionSend = vi.fn(async (method: string) => + method === "Page.getFrameTree" + ? { frameTree: { frame: { loaderId: "LOADER_SAME_URL" } } } + : {}, + ); + const sessionDetach = vi.fn(async () => {}); + const page = { + context: () => ({ + newCDPSession: vi.fn(async () => ({ send: sessionSend, detach: sessionDetach })), + }), + }; + + await expect(readMainFrameDocumentIdentityForPage(page as never)).resolves.toBe( + "cdp:LOADER_SAME_URL", + ); + expect(sessionDetach).toHaveBeenCalledTimes(1); + }); + it("marks backend DOM refs on the page", async () => { const sessionSend = vi.fn(async (method: string, params?: Record) => { if (method === "DOM.pushNodesByBackendIdsToFrontend") { diff --git a/extensions/browser/src/browser/pw-session.page-cdp.ts b/extensions/browser/src/browser/pw-session.page-cdp.ts index 335227370e61..e972600d85bd 100644 --- a/extensions/browser/src/browser/pw-session.page-cdp.ts +++ b/extensions/browser/src/browser/pw-session.page-cdp.ts @@ -6,6 +6,7 @@ */ import { uniqueValues } from "openclaw/plugin-sdk/string-coerce-runtime"; import type { CDPSession, Page } from "playwright-core"; +import { readCdpMainFrameDocumentIdentity } from "./cdp-page-session.js"; type PageCdpSend = (method: string, params?: Record) => Promise; type MarkBackendDomRef = { ref: string; backendDOMNodeId: number }; @@ -44,6 +45,24 @@ export async function withPageScopedCdpClient(opts: { }); } +/** Read the browser-owned loader identity for a Playwright page's main frame. */ +export async function readMainFrameDocumentIdentityForPage( + page: Page, +): Promise { + return await withPlaywrightPageCdpSession( + page, + async (session) => + await readCdpMainFrameDocumentIdentity((method, params) => + ( + session.send as unknown as ( + method: string, + params?: Record, + ) => Promise + )(method, params), + ), + ); +} + /** Mark backend DOM node ids on the page with browser ref attributes. */ export async function markBackendDomRefsOnPage(opts: { page: Page; diff --git a/extensions/browser/src/browser/pw-session.ts b/extensions/browser/src/browser/pw-session.ts index 9ebc28717b46..6dc53a033ccd 100644 --- a/extensions/browser/src/browser/pw-session.ts +++ b/extensions/browser/src/browser/pw-session.ts @@ -41,6 +41,7 @@ export { createPageViaPlaywright, focusPageByTargetIdViaPlaywright, forceDisconnectPlaywrightForTarget, + getMainFrameDocumentIdentityViaPlaywright, getObservedBrowserStateViaPlaywright, listPagesViaPlaywright, refLocator, diff --git a/extensions/browser/src/browser/pw-tools-core.interactions.batch.test.ts b/extensions/browser/src/browser/pw-tools-core.interactions.batch.test.ts index 370f5bf0bde6..8764ab90ad08 100644 --- a/extensions/browser/src/browser/pw-tools-core.interactions.batch.test.ts +++ b/extensions/browser/src/browser/pw-tools-core.interactions.batch.test.ts @@ -4,10 +4,16 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; let page: { evaluate: ReturnType; keyboard: { press: ReturnType }; + isClosed: ReturnType; + mainFrame: ReturnType; mouse: { click: ReturnType }; + off: ReturnType; + on: ReturnType; url: ReturnType; } | null = null; let locator: Record> | null = null; +let setPageClosed: (closed: boolean) => void = () => {}; +let setPageUrl: (url: string) => void = () => {}; const getPageForTargetId = vi.fn(async () => { if (!page) { @@ -19,6 +25,7 @@ const ensurePageState = vi.fn(() => {}); const assertPageNavigationCompletedSafely = vi.fn(async () => {}); const forceDisconnectPlaywrightForTarget = vi.fn(async () => {}); const isBrowserObservedDialogBlockedError = vi.fn(() => false); +const isPolicyDenyNavigationError = vi.fn(() => false); const markObservedDialogsHandledRemotelyForPage = vi.fn(() => ({})); const refLocator = vi.fn(() => { if (!locator) { @@ -47,6 +54,7 @@ vi.mock("./pw-session.js", () => ({ forceDisconnectPlaywrightForTarget, getPageForTargetId, isBrowserObservedDialogBlockedError, + isPolicyDenyNavigationError, markObservedDialogsHandledRemotelyForPage, refLocator, restoreRoleRefsForTarget, @@ -76,25 +84,204 @@ describe("batchViaPlaywright", () => { beforeEach(() => { vi.clearAllMocks(); let currentUrl = "https://example.com"; - const navigate = vi.fn(async () => { - currentUrl = "https://93.184.216.34/target"; - }); + let closed = false; + setPageClosed = (next) => { + closed = next; + }; + const frameNavigatedHandlers = new Set<(frame: unknown) => void>(); + const mainFrame = { url: () => currentUrl }; + setPageUrl = (next) => { + currentUrl = next; + for (const handler of frameNavigatedHandlers) { + handler(mainFrame); + } + }; page = { - evaluate: navigate, - keyboard: { press: navigate }, - mouse: { click: navigate }, + evaluate: vi.fn(async () => {}), + isClosed: vi.fn(() => closed), + keyboard: { press: vi.fn(async () => {}) }, + mainFrame: vi.fn(() => mainFrame), + mouse: { click: vi.fn(async () => {}) }, + off: vi.fn((event: string, handler: (frame: unknown) => void) => { + if (event === "framenavigated") { + frameNavigatedHandlers.delete(handler); + } + }), + on: vi.fn((event: string, handler: (frame: unknown) => void) => { + if (event === "framenavigated") { + frameNavigatedHandlers.add(handler); + } + }), url: vi.fn(() => currentUrl), }; locator = { - click: navigate, - dragTo: navigate, - fill: navigate, - hover: navigate, - press: navigate, - scrollIntoViewIfNeeded: navigate, - selectOption: navigate, - setChecked: navigate, + click: vi.fn(async () => {}), + dragTo: vi.fn(async () => {}), + fill: vi.fn(async () => {}), + hover: vi.fn(async () => {}), + press: vi.fn(async () => {}), + scrollIntoViewIfNeeded: vi.fn(async () => {}), + selectOption: vi.fn(async () => {}), + setChecked: vi.fn(async () => {}), }; + closePageViaPlaywright.mockImplementation(async () => setPageClosed(true)); + }); + + it("aborts remaining actions after a navigation", async () => { + locator!.click!.mockImplementationOnce(() => { + setPageUrl("https://example.com/next"); + }); + + const result = await batchViaPlaywright({ + cdpUrl: "http://127.0.0.1:9222", + targetId: "tab-1", + actions: [ + { kind: "click", ref: "1" }, + { kind: "hover", ref: "2" }, + { kind: "press", key: "Enter" }, + ], + }); + + expect(result).toEqual({ + results: [{ ok: true, navigated: true, url: "https://example.com/next" }], + aborted: { + reason: "navigation", + afterAction: 1, + url: "https://example.com/next", + skipped: 2, + }, + }); + expect(locator!.hover).not.toHaveBeenCalled(); + expect(page!.keyboard.press).not.toHaveBeenCalled(); + }); + + it("aborts remaining actions after a same-URL reload", async () => { + locator!.click!.mockImplementationOnce(() => { + setPageUrl("https://example.com"); + }); + + const result = await batchViaPlaywright({ + cdpUrl: "http://127.0.0.1:9222", + targetId: "tab-1", + actions: [ + { kind: "click", ref: "1" }, + { kind: "hover", ref: "2" }, + ], + }); + + expect(result).toEqual({ + results: [{ ok: true, navigated: true, url: "https://example.com" }], + aborted: { + reason: "navigation", + afterAction: 1, + url: "https://example.com", + skipped: 1, + }, + }); + expect(locator!.hover).not.toHaveBeenCalled(); + expect(page!.off).toHaveBeenCalledWith("framenavigated", expect.any(Function)); + }); + + it("aborts when a navigation commits after an action settles but before the next dispatch", async () => { + let closedChecks = 0; + page!.isClosed.mockImplementation(() => { + closedChecks += 1; + if (closedChecks === 2) { + setPageUrl("https://example.com/late"); + } + return false; + }); + + const result = await batchViaPlaywright({ + cdpUrl: "http://127.0.0.1:9222", + targetId: "tab-1", + actions: [ + { kind: "click", ref: "1" }, + { kind: "hover", ref: "2" }, + ], + }); + + expect(result).toEqual({ + results: [{ ok: true, navigated: true, url: "https://example.com/late" }], + aborted: { + reason: "navigation", + afterAction: 1, + url: "https://example.com/late", + skipped: 1, + }, + }); + expect(locator!.hover).not.toHaveBeenCalled(); + }); + + it("runs every action when the page URL stays unchanged", async () => { + const result = await batchViaPlaywright({ + cdpUrl: "http://127.0.0.1:9222", + targetId: "tab-1", + actions: [ + { kind: "click", ref: "1" }, + { kind: "hover", ref: "2" }, + { kind: "press", key: "Enter" }, + ], + }); + + expect(result).toEqual({ results: [{ ok: true }, { ok: true }, { ok: true }] }); + }); + + it("keeps stopOnError=false until a later navigation aborts the batch", async () => { + locator!.click!.mockRejectedValueOnce(new Error("click failed")); + locator!.hover!.mockImplementationOnce(() => { + setPageUrl("https://example.com/next"); + }); + + const result = await batchViaPlaywright({ + cdpUrl: "http://127.0.0.1:9222", + targetId: "tab-1", + stopOnError: false, + actions: [ + { kind: "click", ref: "1" }, + { kind: "hover", ref: "2" }, + { kind: "press", key: "Enter" }, + ], + }); + + expect(result).toEqual({ + results: [ + { ok: false, error: "click failed" }, + { ok: true, navigated: true, url: "https://example.com/next" }, + ], + aborted: { + reason: "navigation", + afterAction: 2, + url: "https://example.com/next", + skipped: 1, + }, + }); + expect(page!.keyboard.press).not.toHaveBeenCalled(); + }); + + it("aborts when the page closes during an action", async () => { + locator!.click!.mockImplementationOnce(() => setPageClosed(true)); + + const result = await batchViaPlaywright({ + cdpUrl: "http://127.0.0.1:9222", + targetId: "tab-1", + stopOnError: false, + actions: [ + { kind: "click", ref: "1" }, + { kind: "hover", ref: "2" }, + ], + }); + + expect(result).toEqual({ + results: [{ ok: true }], + aborted: { + reason: "closed", + afterAction: 1, + url: "https://example.com", + skipped: 1, + }, + }); + expect(locator!.hover).not.toHaveBeenCalled(); }); it("propagates evaluate timeouts through batched execution", async () => { diff --git a/extensions/browser/src/browser/pw-tools-core.interactions.ts b/extensions/browser/src/browser/pw-tools-core.interactions.ts index d1ed0acac814..68aa03e13021 100644 --- a/extensions/browser/src/browser/pw-tools-core.interactions.ts +++ b/extensions/browser/src/browser/pw-tools-core.interactions.ts @@ -16,6 +16,7 @@ import { resolveActInteractionTimeoutMs, resolveActWaitTimeoutMs, } from "./act-policy.js"; +import type { BrowserBatchAbort, BrowserBatchActionResult } from "./client-actions-types.js"; import type { BrowserActRequest, BrowserFormField } from "./client-actions.types.js"; import type { BrowserDownloadResult } from "./download-types.js"; import { normalizeBrowserEvaluateFunctionSource } from "./evaluate-source.js"; @@ -1953,7 +1954,8 @@ export async function executeActViaPlaywright( } & BrowserNavigationPolicyOptions, ): Promise<{ result?: unknown; - results?: Array<{ ok: boolean; error?: string }>; + results?: BrowserBatchActionResult[]; + aborted?: BrowserBatchAbort; blockedByDialog?: boolean; browserState?: unknown; downloads?: BrowserDownloadResult[]; @@ -1995,6 +1997,7 @@ export async function executeActViaPlaywright( const batch = await batchViaPlaywright({ cdpUrl: opts.cdpUrl, targetId: opts.targetId, + page, ...navigationPolicy, actions: opts.action.actions, stopOnError: opts.action.stopOnError, @@ -2004,6 +2007,7 @@ export async function executeActViaPlaywright( const newDownloads = await drainDownloads(); return { results: batch.results, + ...(batch.aborted ? { aborted: batch.aborted } : {}), ...(newDownloads ? { downloads: newDownloads } : {}), }; } @@ -2059,13 +2063,14 @@ export async function batchViaPlaywright( opts: { cdpUrl: string; targetId?: string; + page?: Page; actions: BrowserActRequest[]; stopOnError?: boolean; evaluateEnabled?: boolean; depth?: number; signal?: AbortSignal; } & BrowserNavigationPolicyOptions, -): Promise<{ results: Array<{ ok: boolean; error?: string }> }> { +): Promise<{ results: BrowserBatchActionResult[]; aborted?: BrowserBatchAbort }> { const navigationPolicy = interactionNavigationPolicy(opts); const depth = opts.depth ?? 0; if (depth > ACT_MAX_BATCH_DEPTH) { @@ -2074,37 +2079,101 @@ export async function batchViaPlaywright( if (opts.actions.length > ACT_MAX_BATCH_ACTIONS) { throw new Error(`Batch exceeds maximum of ${ACT_MAX_BATCH_ACTIONS} actions`); } - const results: Array<{ ok: boolean; error?: string }> = []; - for (const action of opts.actions) { - if (opts.signal?.aborted) { - throw opts.signal.reason ?? new Error("aborted"); + const page = opts.page ?? (await getPageForTargetId(opts)); + const results: BrowserBatchActionResult[] = []; + const finishAborted = ( + reason: BrowserBatchAbort["reason"], + afterAction: number, + url: string, + skipped: number, + ) => + skipped === 0 + ? { results } + : { results, aborted: { reason, afterAction, url, skipped } satisfies BrowserBatchAbort }; + let mainFrameNavigations = 0; + let navigationsAtLastDispatch = 0; + const currentMainFrameUrl = () => page.mainFrame?.().url() ?? page.url(); + const onFrameNavigated = (frame: Frame) => { + if (frame === page.mainFrame?.()) { + mainFrameNavigations += 1; } - try { - await executeSingleAction( - action, - opts.cdpUrl, - opts.targetId, - opts.evaluateEnabled, - navigationPolicy, - depth, - opts.signal, - ); - results.push({ ok: true }); - } catch (err) { - if (isBrowserObservedDialogBlockedError(err)) { - throw err; + }; + const finishNavigation = (afterAction: number, skipped: number) => { + const url = currentMainFrameUrl(); + const lastResult = results.at(-1); + if (lastResult) { + results[results.length - 1] = { ...lastResult, navigated: true, url }; + } + return finishAborted("navigation", afterAction, url, skipped); + }; + + // Snapshot refs are document-scoped, so any committed main-frame navigation + // ends the batch. A commit after the next action dispatch is inherently unguardable; + // callers that expect navigation can use separate act calls as the escape hatch. + page.on?.("framenavigated", onFrameNavigated); + try { + for (const [index, action] of opts.actions.entries()) { + if (opts.signal?.aborted) { + throw opts.signal.reason ?? new Error("aborted"); } - if (isPolicyDenyNavigationError(err)) { - throw err; + if (mainFrameNavigations > navigationsAtLastDispatch) { + return finishNavigation(index, opts.actions.length - index); } - const message = formatErrorMessage(err); - results.push({ ok: false, error: message }); - if (opts.stopOnError !== false) { - break; + if (page.isClosed?.()) { + return finishAborted("closed", index, currentMainFrameUrl(), opts.actions.length - index); + } + navigationsAtLastDispatch = mainFrameNavigations; + try { + await executeSingleAction( + action, + opts.cdpUrl, + opts.targetId, + opts.evaluateEnabled, + navigationPolicy, + depth, + opts.signal, + ); + results.push({ ok: true }); + if (page.isClosed?.()) { + return finishAborted( + "closed", + index + 1, + currentMainFrameUrl(), + opts.actions.length - index - 1, + ); + } + if (mainFrameNavigations > navigationsAtLastDispatch) { + return finishNavigation(index + 1, opts.actions.length - index - 1); + } + } catch (err) { + if (isBrowserObservedDialogBlockedError(err)) { + throw err; + } + if (isPolicyDenyNavigationError(err)) { + throw err; + } + const message = formatErrorMessage(err); + results.push({ ok: false, error: message }); + if (page.isClosed?.()) { + return finishAborted( + "closed", + index + 1, + currentMainFrameUrl(), + opts.actions.length - index - 1, + ); + } + if (mainFrameNavigations > navigationsAtLastDispatch) { + return finishNavigation(index + 1, opts.actions.length - index - 1); + } + if (opts.stopOnError !== false) { + break; + } } } + return { results }; + } finally { + page.off?.("framenavigated", onFrameNavigated); } - return { results }; } /* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ diff --git a/extensions/browser/src/browser/pw-tools-core.snapshot.ts b/extensions/browser/src/browser/pw-tools-core.snapshot.ts index 9eb88e664be9..7a0fde68d213 100644 --- a/extensions/browser/src/browser/pw-tools-core.snapshot.ts +++ b/extensions/browser/src/browser/pw-tools-core.snapshot.ts @@ -24,6 +24,7 @@ import { buildRoleSnapshotFromAiSnapshot, buildRoleSnapshotFromAriaSnapshot, finalizeRoleSnapshot, + type RoleSnapshotIdentityMode, type RoleSnapshotOptions, type RoleRefMap, } from "./pw-role-snapshot.js"; @@ -260,7 +261,13 @@ export async function snapshotAiViaPlaywright(opts: { maxChars?: number; urls?: boolean; ssrfPolicy?: SsrFPolicy; -}): Promise<{ snapshot: string; truncated?: boolean; refs: RoleRefMap }> { + delta?: { mode: RoleSnapshotIdentityMode; previousKeys?: ReadonlySet }; +}): Promise<{ + snapshot: string; + truncated?: boolean; + refs: RoleRefMap; + newElements?: number; +}> { const page = await prepareSnapshotPageViaPlaywright({ cdpUrl: opts.cdpUrl, targetId: opts.targetId, @@ -282,6 +289,7 @@ export async function snapshotAiViaPlaywright(opts: { snapshot, refs: built.refs, maxChars: opts.maxChars, + delta: opts.delta, }); assertSnapshotFrameCurrent(isFrameCurrent); storeRoleRefsForTarget({ @@ -335,11 +343,13 @@ async function finalizeRoleSnapshotViaPlaywright(params: { built: { snapshot: string; refs: RoleRefMap }; urls?: boolean; maxChars?: number; + delta?: { mode: RoleSnapshotIdentityMode; previousKeys?: ReadonlySet }; }): Promise<{ snapshot: string; truncated?: boolean; refs: RoleRefMap; stats: { lines: number; chars: number; refs: number; interactive: number }; + newElements?: number; }> { const snapshot = params.urls ? appendSnapshotUrls(params.built.snapshot, await collectSnapshotUrls(params.page)) @@ -351,6 +361,7 @@ async function finalizeRoleSnapshotViaPlaywright(params: { snapshot, refs: params.built.refs, maxChars: params.maxChars, + delta: params.delta, }); storeRoleRefsForTarget({ page: params.page, @@ -376,11 +387,13 @@ export async function snapshotRoleViaPlaywright(opts: { maxChars?: number; timeoutMs?: number; ssrfPolicy?: SsrFPolicy; + delta?: { mode: RoleSnapshotIdentityMode; previousKeys?: ReadonlySet }; }): Promise<{ snapshot: string; truncated?: boolean; refs: Record; stats: { lines: number; chars: number; refs: number; interactive: number }; + newElements?: number; }> { const page = await prepareSnapshotPageViaPlaywright({ cdpUrl: opts.cdpUrl, @@ -411,6 +424,7 @@ export async function snapshotRoleViaPlaywright(opts: { mode: "aria", urls: opts.urls, maxChars: opts.maxChars, + delta: opts.delta, }); }, }); @@ -456,6 +470,7 @@ export async function snapshotRoleViaPlaywright(opts: { mode: "role", urls: opts.urls, maxChars: opts.maxChars, + delta: opts.delta, }); }, }); diff --git a/extensions/browser/src/browser/routes/agent.act.ts b/extensions/browser/src/browser/routes/agent.act.ts index baa591ad2b66..9ee6d6000aa1 100644 --- a/extensions/browser/src/browser/routes/agent.act.ts +++ b/extensions/browser/src/browser/routes/agent.act.ts @@ -31,6 +31,7 @@ import { } from "../navigation-guard.js"; import { getBrowserProfileCapabilities } from "../profile-capabilities.js"; import type { BrowserRouteContext } from "../server-context.js"; +import { clearSnapshotKeysForTab } from "../snapshot-delta-cache.js"; import { matchBrowserUrlPattern } from "../url-pattern.js"; import { registerBrowserAgentActDownloadRoutes } from "./agent.act.download.js"; import { @@ -691,6 +692,7 @@ export function registerBrowserAgentActRoutes( ...existingSessionCallOptions, exactTargetId: true, }); + clearSnapshotKeysForTab(ctx, profileCtx.profile.name, tab.targetId); return await jsonOk(); case "batch": return jsonActError( @@ -721,10 +723,17 @@ export function registerBrowserAgentActRoutes( }); } const downloads = result.downloads; + if (action.kind === "close" || result.aborted?.reason === "closed") { + clearSnapshotKeysForTab(ctx, profileCtx.profile.name, tab.targetId); + } switch (action.kind) { case "batch": return await jsonOk( - { results: result.results ?? [], ...(downloads ? { downloads } : {}) }, + { + results: result.results ?? [], + ...(result.aborted ? { aborted: result.aborted } : {}), + ...(downloads ? { downloads } : {}), + }, { resolveCurrentTarget: true }, ); case "evaluate": diff --git a/extensions/browser/src/browser/routes/agent.existing-session.test.ts b/extensions/browser/src/browser/routes/agent.existing-session.test.ts index b39af591f480..349fe4b652c2 100644 --- a/extensions/browser/src/browser/routes/agent.existing-session.test.ts +++ b/extensions/browser/src/browser/routes/agent.existing-session.test.ts @@ -206,6 +206,35 @@ describe("existing-session browser routes", () => { expect(chromeMcpMocks.takeChromeMcpScreenshot).toHaveBeenCalled(); }); + it("omits deltas for existing-session snapshots without stable document identity", async () => { + chromeMcpMocks.takeChromeMcpSnapshot + .mockResolvedValueOnce({ + id: "root-1", + role: "document", + name: "Example", + children: [{ id: "save-1", role: "button", name: "Save" }], + }) + .mockResolvedValueOnce({ + id: "root-2", + role: "document", + name: "Example", + children: [ + { id: "save-2", role: "button", name: "Save" }, + { id: "alert-2", role: "alert", name: "Required" }, + ], + }); + const handler = getSnapshotGetHandler(); + const first = createBrowserRouteResponse(); + const second = createBrowserRouteResponse(); + + await handler?.({ params: {}, query: { format: "ai" } }, first.res); + await handler?.({ params: {}, query: { format: "ai" } }, second.res); + + const body = requireRecord(second.body, "second snapshot body"); + expect(body.snapshot).not.toContain("[new]"); + expect(body.newElements).toBeUndefined(); + }); + it("labels and returns only Chrome MCP refs inside the final snapshot budget", async () => { chromeMcpMocks.takeChromeMcpSnapshot.mockResolvedValueOnce({ id: "root", diff --git a/extensions/browser/src/browser/routes/agent.snapshot.local-managed.test.ts b/extensions/browser/src/browser/routes/agent.snapshot.local-managed.test.ts index 101f0d47fb0f..6e47a49c6af6 100644 --- a/extensions/browser/src/browser/routes/agent.snapshot.local-managed.test.ts +++ b/extensions/browser/src/browser/routes/agent.snapshot.local-managed.test.ts @@ -20,10 +20,13 @@ const routeState = vi.hoisted(() => ({ })); const cdpMocks = vi.hoisted(() => ({ + getMainFrameDocumentIdentityViaCdp: vi.fn<() => Promise>( + async () => "cdp:test-document", + ), snapshotAria: vi.fn(async () => ({ nodes: [{ ref: "1", role: "link", name: "private", depth: 0 }], })), - snapshotRoleViaCdp: vi.fn(async () => ({ + snapshotRoleViaCdp: vi.fn(async (_opts: unknown) => ({ snapshot: '- link "private" [ref=e1]', refs: { e1: { role: "link", name: "private" } }, stats: { lines: 1, chars: 25, refs: 1, interactive: 1 }, @@ -40,6 +43,7 @@ const navigationGuardMocks = vi.hoisted(() => ({ vi.mock("../cdp.js", () => ({ captureScreenshot: vi.fn(), + getMainFrameDocumentIdentityViaCdp: cdpMocks.getMainFrameDocumentIdentityViaCdp, snapshotAria: cdpMocks.snapshotAria, snapshotRoleViaCdp: cdpMocks.snapshotRoleViaCdp, })); @@ -115,6 +119,7 @@ function getSnapshotGetHandler() { describe("local-managed browser snapshot routes", () => { beforeEach(() => { routeState.profileCtx.ensureTabAvailable.mockClear(); + cdpMocks.getMainFrameDocumentIdentityViaCdp.mockReset().mockResolvedValue("cdp:test-document"); cdpMocks.snapshotAria.mockClear(); cdpMocks.snapshotRoleViaCdp.mockClear(); navigationGuardMocks.assertBrowserNavigationResultAllowed.mockClear(); @@ -171,4 +176,53 @@ describe("local-managed browser snapshot routes", () => { expect.objectContaining({ maxChars: 123 }), ); }); + + it("rejects a snapshot when the main-frame loader changes during capture", async () => { + navigationGuardMocks.assertBrowserNavigationResultAllowed.mockResolvedValueOnce(undefined); + cdpMocks.getMainFrameDocumentIdentityViaCdp + .mockResolvedValueOnce("cdp:before") + .mockResolvedValueOnce("cdp:after"); + const handler = getSnapshotGetHandler(); + const response = createBrowserRouteResponse(); + + await handler?.({ params: {}, query: { format: "ai", interactive: "true" } }, response.res); + + expect(response.statusCode).toBe(400); + expect(response.body).toEqual({ + error: "Frame changed while its browser snapshot was being captured; retry.", + }); + }); + + it("disables deltas when no stable document identity is available", async () => { + navigationGuardMocks.assertBrowserNavigationResultAllowed.mockResolvedValue(undefined); + cdpMocks.getMainFrameDocumentIdentityViaCdp.mockResolvedValue(undefined); + const handler = getSnapshotGetHandler(); + const first = createBrowserRouteResponse(); + const second = createBrowserRouteResponse(); + + await handler?.({ params: {}, query: { format: "ai", interactive: "true" } }, first.res); + await handler?.({ params: {}, query: { format: "ai", interactive: "true" } }, second.res); + + const calls = cdpMocks.snapshotRoleViaCdp.mock.calls; + expect(calls).toHaveLength(2); + expect(calls[0]?.[0]).toMatchObject({ delta: undefined }); + expect(calls[1]?.[0]).toMatchObject({ delta: undefined }); + expect(second.body).not.toHaveProperty("newElements"); + expect(second.body).not.toHaveProperty("snapshot", expect.stringContaining("[new]")); + }); + + it("reuses delta keys when the stable document identity is unchanged", async () => { + navigationGuardMocks.assertBrowserNavigationResultAllowed.mockResolvedValue(undefined); + const handler = getSnapshotGetHandler(); + const first = createBrowserRouteResponse(); + const second = createBrowserRouteResponse(); + + await handler?.({ params: {}, query: { format: "ai", interactive: "true" } }, first.res); + await handler?.({ params: {}, query: { format: "ai", interactive: "true" } }, second.res); + + const secondCall = cdpMocks.snapshotRoleViaCdp.mock.calls[1]?.[0]; + expect(secondCall).toMatchObject({ + delta: { mode: "role", previousKeys: expect.any(Set) }, + }); + }); }); diff --git a/extensions/browser/src/browser/routes/agent.snapshot.timeout.test.ts b/extensions/browser/src/browser/routes/agent.snapshot.timeout.test.ts index fdd323a27a4b..29dbd41c16c0 100644 --- a/extensions/browser/src/browser/routes/agent.snapshot.timeout.test.ts +++ b/extensions/browser/src/browser/routes/agent.snapshot.timeout.test.ts @@ -4,6 +4,7 @@ import { createBrowserRouteApp, createBrowserRouteResponse } from "./test-helper const cdpMocks = vi.hoisted(() => ({ captureScreenshot: vi.fn(), + getMainFrameDocumentIdentityViaCdp: vi.fn(async () => "cdp:test-document"), snapshotAria: vi.fn(async () => ({ nodes: [] })), snapshotRoleViaCdp: vi.fn(async () => ({ snapshot: "button Continue", @@ -33,6 +34,7 @@ const profileContext = vi.hoisted(() => ({ vi.mock("../cdp.js", () => ({ captureScreenshot: cdpMocks.captureScreenshot, + getMainFrameDocumentIdentityViaCdp: cdpMocks.getMainFrameDocumentIdentityViaCdp, snapshotAria: cdpMocks.snapshotAria, snapshotRoleViaCdp: cdpMocks.snapshotRoleViaCdp, })); diff --git a/extensions/browser/src/browser/routes/agent.snapshot.ts b/extensions/browser/src/browser/routes/agent.snapshot.ts index 7b21d349d626..9872389baa6f 100644 --- a/extensions/browser/src/browser/routes/agent.snapshot.ts +++ b/extensions/browser/src/browser/routes/agent.snapshot.ts @@ -8,7 +8,12 @@ import path from "node:path"; import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime"; import { getImageMetadata } from "../../media/media-services.js"; import { ensureMediaDir, saveMediaBuffer } from "../../media/store.js"; -import { captureScreenshot, snapshotAria, snapshotRoleViaCdp } from "../cdp.js"; +import { + captureScreenshot, + getMainFrameDocumentIdentityViaCdp, + snapshotAria, + snapshotRoleViaCdp, +} from "../cdp.js"; import { evaluateChromeMcpScript, navigateChromeMcpPage, @@ -27,7 +32,7 @@ import { assertBrowserNavigationResultAllowed, } from "../navigation-guard.js"; import { getBrowserProfileCapabilities } from "../profile-capabilities.js"; -import { finalizeRoleSnapshot } from "../pw-role-snapshot.js"; +import { finalizeRoleSnapshot, type RoleRefMap } from "../pw-role-snapshot.js"; import type { AnnotationItem } from "../screenshot-annotate.js"; import { scaleAnnotations } from "../screenshot-annotate.js"; import { @@ -36,6 +41,11 @@ import { normalizeBrowserScreenshot, } from "../screenshot.js"; import type { BrowserRouteContext } from "../server-context.js"; +import { + getPreviousSnapshotKeys, + recordSnapshotKeys, + type SnapshotDeltaFamily, +} from "../snapshot-delta-cache.js"; import { appendSnapshotUrls, type SnapshotUrlEntry } from "../snapshot-urls.js"; import { normalizeBrowserTimerDelayMs } from "../timer-delay.js"; import { @@ -597,16 +607,56 @@ export function registerBrowserAgentSnapshotRoutes( ...ssrfPolicyOpts, }); } - let observedBrowserState: unknown; - if (!usesChromeMcp && pwModule) { - observedBrowserState = await pwModule - .getObservedBrowserStateViaPlaywright({ - cdpUrl: profileCtx.profile.cdpUrl, - targetId: tab.targetId, - ssrfPolicy: ctx.state().resolved.ssrfPolicy, - }) - .catch(() => undefined); - } + const deltaFamily: SnapshotDeltaFamily | undefined = + plan.format === "ai" + ? { + identity: usesChromeMcp + ? "aria" + : plan.wantsRoleSnapshot + ? plan.refsMode === "aria" + ? "aria" + : "role" + : pwModule + ? "aria" + : "role", + interactive: plan.interactive, + compact: plan.compact, + depth: plan.depth, + selector: plan.selectorValue, + frame: plan.frameSelectorValue, + urls: plan.urls, + maxChars: plan.resolvedMaxChars, + } + : undefined; + const createDeltaState = (documentIdentity?: string) => { + const previousKeys = + deltaFamily && documentIdentity + ? getPreviousSnapshotKeys(ctx, { + profile: profileCtx.profile.name, + targetId: tab.targetId, + documentIdentity, + family: deltaFamily, + }) + : undefined; + return { + delta: + deltaFamily && previousKeys !== undefined + ? { mode: deltaFamily.identity, previousKeys } + : undefined, + record: (refs: RoleRefMap) => { + if (!deltaFamily || !documentIdentity) { + return; + } + recordSnapshotKeys(ctx, { + profile: profileCtx.profile.name, + targetId: tab.targetId, + documentIdentity, + family: deltaFamily, + refs, + }); + }, + }; + }; if (usesChromeMcp) { const operation: ChromeMcpSnapshotOperation = { profileName: profileCtx.profile.name, @@ -625,6 +675,7 @@ export function registerBrowserAgentSnapshotRoutes( nodes: flattenChromeMcpSnapshotToAriaNodes(snapshot, plan.limit), }); } + const deltaState = createDeltaState(); const built = buildAiSnapshotFromChromeMcpSnapshot({ root: snapshot, options: { @@ -645,6 +696,7 @@ export function registerBrowserAgentSnapshotRoutes( const finalized = finalizeRoleSnapshot({ ...builtWithUrls, maxChars: plan.resolvedMaxChars, + delta: deltaState.delta, }); if (plan.labels) { const refs = Object.keys(finalized.refs); @@ -668,6 +720,7 @@ export function registerBrowserAgentSnapshotRoutes( "browser", DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES, ); + deltaState.record(finalized.refs); return res.json({ ok: true, format: "ai", @@ -684,6 +737,7 @@ export function registerBrowserAgentSnapshotRoutes( await clearChromeMcpOverlay(operation); } } + deltaState.record(finalized.refs); return res.json({ ok: true, format: "ai", @@ -692,6 +746,18 @@ export function registerBrowserAgentSnapshotRoutes( ...finalized, }); } + const readPlaywrightDocumentIdentity = + pwModule?.getMainFrameDocumentIdentityViaPlaywright; + let observedBrowserState: unknown; + if (pwModule) { + observedBrowserState = await pwModule + .getObservedBrowserStateViaPlaywright({ + cdpUrl: profileCtx.profile.cdpUrl, + targetId: tab.targetId, + ssrfPolicy: ctx.state().resolved.ssrfPolicy, + }) + .catch(() => undefined); + } if (hasPendingDialogs(observedBrowserState)) { return res.json({ ok: true, @@ -703,6 +769,37 @@ export function registerBrowserAgentSnapshotRoutes( ...(plan.format === "aria" ? { nodes: [] } : { snapshot: "", refs: {} }), }); } + const readDocumentIdentity = async (): Promise => { + if (!deltaFamily) { + return undefined; + } + const playwrightIdentity = readPlaywrightDocumentIdentity + ? await readPlaywrightDocumentIdentity({ + cdpUrl: profileCtx.profile.cdpUrl, + targetId: tab.targetId, + }).catch(() => undefined) + : undefined; + if (playwrightIdentity || !tab.wsUrl) { + return playwrightIdentity; + } + return await getMainFrameDocumentIdentityViaCdp({ + wsUrl: tab.wsUrl, + timeoutMs: plan.timeoutMs, + }).catch(() => undefined); + }; + const initialDocumentIdentity = await readDocumentIdentity(); + const deltaState = createDeltaState(initialDocumentIdentity); + const assertDocumentIdentityUnchanged = async () => { + if (!initialDocumentIdentity) { + return; + } + const finalDocumentIdentity = await readDocumentIdentity(); + if (finalDocumentIdentity !== initialDocumentIdentity) { + throw new Error( + "Frame changed while its browser snapshot was being captured; retry.", + ); + } + }; if (plan.format === "ai") { const roleSnapshotArgs = { cdpUrl: profileCtx.profile.cdpUrl, @@ -719,6 +816,7 @@ export function registerBrowserAgentSnapshotRoutes( compact: plan.compact ?? undefined, maxDepth: plan.depth ?? undefined, }, + delta: deltaState.delta, }; const cdpRoleSnapshot = async () => { @@ -738,6 +836,7 @@ export function registerBrowserAgentSnapshotRoutes( compact: plan.compact ?? undefined, maxDepth: plan.depth ?? undefined, }, + delta: deltaState.delta, }); }; @@ -764,6 +863,7 @@ export function registerBrowserAgentSnapshotRoutes( ...(typeof plan.resolvedMaxChars === "number" ? { maxChars: plan.resolvedMaxChars } : {}), + delta: deltaState.delta, }) : await cdpRoleSnapshot(); if (!snap) { @@ -801,6 +901,8 @@ export function registerBrowserAgentSnapshotRoutes( DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES, ); const imageType = normalized.contentType?.includes("jpeg") ? "jpeg" : "png"; + await assertDocumentIdentityUnchanged(); + deltaState.record(snap.refs ?? {}); return res.json({ ok: true, format: plan.format, @@ -819,6 +921,8 @@ export function registerBrowserAgentSnapshotRoutes( }); } + await assertDocumentIdentityUnchanged(); + deltaState.record(snap.refs ?? {}); return res.json({ ok: true, format: plan.format, diff --git a/extensions/browser/src/browser/routes/tabs.ts b/extensions/browser/src/browser/routes/tabs.ts index f1819fe040ec..915b5c0575c1 100644 --- a/extensions/browser/src/browser/routes/tabs.ts +++ b/extensions/browser/src/browser/routes/tabs.ts @@ -17,6 +17,7 @@ import { import { getBrowserProfileCapabilities } from "../profile-capabilities.js"; import type { BrowserRouteContext, ProfileContext } from "../server-context.js"; import { isProfileRestartRequiredError } from "../server-context.lifecycle.js"; +import { clearSnapshotKeysForTab } from "../snapshot-delta-cache.js"; import { resolveTargetIdFromTabs } from "../target-id.js"; import { browserNavigationPolicyForProfile, resolveProfileContext } from "./agent.shared.js"; import { readRouteNonNegativeInteger } from "./route-numeric.js"; @@ -321,6 +322,7 @@ export function registerBrowserTabRoutes(app: BrowserRouteRegistrar, ctx: Browse targetId, mutate: async (profileCtx, id) => { await profileCtx.closeTab(id, targetIdMode === "raw" ? { exactTargetId: true } : undefined); + clearSnapshotKeysForTab(ctx, profileCtx.profile.name, id); }, }); }); @@ -397,6 +399,7 @@ export function registerBrowserTabRoutes(app: BrowserRouteRegistrar, ctx: Browse throw new BrowserTabNotFoundError(); } await profileCtx.closeTab(target.targetId, { exactTargetId: true }); + clearSnapshotKeysForTab(ctx, profileCtx.profile.name, target.targetId); return { ok: true, targetId: target.targetId }; } diff --git a/extensions/browser/src/browser/server.control-server.test-harness.ts b/extensions/browser/src/browser/server.control-server.test-harness.ts index e9b423686da1..02e6194a53b6 100644 --- a/extensions/browser/src/browser/server.control-server.test-harness.ts +++ b/extensions/browser/src/browser/server.control-server.test-harness.ts @@ -107,6 +107,7 @@ const cdpMocks = vi.hoisted(() => ({ createTargetViaCdp: vi.fn<() => Promise<{ targetId: string }>>(async () => { throw new Error("cdp disabled"); }), + getMainFrameDocumentIdentityViaCdp: vi.fn(async () => "cdp:test-document"), snapshotAria: vi.fn(async () => ({ nodes: [{ ref: "1", role: "link", name: "x", depth: 0 }], })), @@ -120,11 +121,13 @@ const cdpMocks = vi.hoisted(() => ({ /** Returns mocked CDP functions used by Browser control-server tests. */ export function getCdpMocks(): { createTargetViaCdp: MockFn; + getMainFrameDocumentIdentityViaCdp: MockFn; snapshotAria: MockFn; snapshotRoleViaCdp: MockFn; } { return cdpMocks as unknown as { createTargetViaCdp: MockFn; + getMainFrameDocumentIdentityViaCdp: MockFn; snapshotAria: MockFn; snapshotRoleViaCdp: MockFn; }; @@ -211,6 +214,7 @@ const pwMocks = vi.hoisted(() => { getObservedBrowserStateViaPlaywright: vi.fn(async () => ({ dialogs: { pending: [], recent: [] }, })), + getMainFrameDocumentIdentityViaPlaywright: vi.fn(async () => "pw:test-document"), getPageErrorsViaPlaywright: vi.fn(async () => ({ errors: [] })), highlightViaPlaywright: vi.fn(async (_opts?: unknown) => {}), hoverViaPlaywright: vi.fn(async (_opts?: unknown) => {}), @@ -533,6 +537,7 @@ vi.mock("./chrome.js", () => ({ vi.mock("./cdp.js", () => ({ createTargetViaCdp: cdpMocks.createTargetViaCdp, + getMainFrameDocumentIdentityViaCdp: cdpMocks.getMainFrameDocumentIdentityViaCdp, normalizeCdpWsUrl: vi.fn((wsUrl: string) => wsUrl), snapshotAria: cdpMocks.snapshotAria, snapshotRoleViaCdp: cdpMocks.snapshotRoleViaCdp, diff --git a/extensions/browser/src/browser/snapshot-delta-cache.test.ts b/extensions/browser/src/browser/snapshot-delta-cache.test.ts new file mode 100644 index 000000000000..85a8f71c746c --- /dev/null +++ b/extensions/browser/src/browser/snapshot-delta-cache.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from "vitest"; +import { finalizeRoleSnapshot } from "./pw-role-snapshot.js"; +import type { BrowserRouteContext } from "./server-context.types.js"; +import { + getPreviousSnapshotKeys, + recordSnapshotKeys, + type SnapshotDeltaFamily, +} from "./snapshot-delta-cache.js"; + +const family: SnapshotDeltaFamily = { identity: "role", interactive: true }; + +function createContext(): BrowserRouteContext { + return {} as BrowserRouteContext; +} + +function cacheScope(documentIdentity: string) { + return { profile: "openclaw", targetId: "tab-1", documentIdentity, family }; +} + +describe("snapshot delta cache", () => { + it("starts an unmarked baseline after a same-URL reload", () => { + const ctx = createContext(); + recordSnapshotKeys(ctx, { + ...cacheScope("pw:document-1"), + refs: { e1: { role: "button", name: "Save" } }, + }); + + const previousKeys = getPreviousSnapshotKeys(ctx, cacheScope("pw:document-2")); + const snapshot = finalizeRoleSnapshot({ + snapshot: '- button "Continue" [ref=e1]', + refs: { e1: { role: "button", name: "Continue" } }, + delta: { mode: "role", previousKeys }, + }); + + expect(previousKeys).toBeUndefined(); + expect(snapshot.snapshot).toBe('- button "Continue" [ref=e1]'); + expect(snapshot.newElements).toBeUndefined(); + }); + + it("marks elements added in the same document", () => { + const ctx = createContext(); + const scope = cacheScope("pw:document-1"); + recordSnapshotKeys(ctx, { + ...scope, + refs: { e1: { role: "button", name: "Save" } }, + }); + + const snapshot = finalizeRoleSnapshot({ + snapshot: ['- button "Save" [ref=e7]', '- alert "Required" [ref=e8]'].join("\n"), + refs: { + e7: { role: "button", name: "Save" }, + e8: { role: "alert", name: "Required" }, + }, + delta: { mode: "role", previousKeys: getPreviousSnapshotKeys(ctx, scope) }, + }); + + expect(snapshot.snapshot).toContain('- alert "Required" [ref=e8] [new]'); + expect(snapshot.newElements).toBe(1); + }); +}); diff --git a/extensions/browser/src/browser/snapshot-delta-cache.ts b/extensions/browser/src/browser/snapshot-delta-cache.ts new file mode 100644 index 000000000000..744d5c9dc896 --- /dev/null +++ b/extensions/browser/src/browser/snapshot-delta-cache.ts @@ -0,0 +1,120 @@ +import { + getRoleSnapshotIdentityKeys, + type RoleRefMap, + type RoleSnapshotIdentityMode, +} from "./pw-role-snapshot.js"; +import type { BrowserRouteContext } from "./server-context.types.js"; + +/** + * Process-local snapshot delta state owned by one Browser control-server context. + * Each tab/options family keeps one bounded previous-key slot; restart or tab close drops it. + */ +const SNAPSHOT_DELTA_CACHE_MAX_ENTRIES = 32; + +export type SnapshotDeltaFamily = { + identity: RoleSnapshotIdentityMode; + interactive?: boolean; + compact?: boolean; + depth?: number; + selector?: string; + frame?: string; + urls?: boolean; + maxChars?: number; +}; + +type SnapshotDeltaEntry = { + profile: string; + targetId: string; + documentIdentity: string; + keys: Set; +}; + +const cacheByContext = new WeakMap>(); + +function getCache(ctx: BrowserRouteContext): Map { + const existing = cacheByContext.get(ctx); + if (existing) { + return existing; + } + const cache = new Map(); + cacheByContext.set(ctx, cache); + return cache; +} + +function cacheKey(params: { + profile: string; + targetId: string; + family: SnapshotDeltaFamily; +}): string { + return JSON.stringify([params.profile, params.targetId, params.family]); +} + +export function getPreviousSnapshotKeys( + ctx: BrowserRouteContext, + params: { + profile: string; + targetId: string; + documentIdentity: string; + family: SnapshotDeltaFamily; + }, +): ReadonlySet | undefined { + const cache = getCache(ctx); + const key = cacheKey(params); + const entry = cache.get(key); + if (!entry) { + return undefined; + } + // Delta markers are same-document only. Navigation resets the baseline so a + // replacement document is not reported as a tree full of newly appeared elements. + if (entry.documentIdentity !== params.documentIdentity) { + cache.delete(key); + return undefined; + } + cache.delete(key); + cache.set(key, entry); + return entry.keys; +} + +export function recordSnapshotKeys( + ctx: BrowserRouteContext, + params: { + profile: string; + targetId: string; + documentIdentity: string; + family: SnapshotDeltaFamily; + refs: RoleRefMap; + }, +): void { + const cache = getCache(ctx); + const key = cacheKey(params); + cache.delete(key); + cache.set(key, { + profile: params.profile, + targetId: params.targetId, + documentIdentity: params.documentIdentity, + keys: getRoleSnapshotIdentityKeys(params.refs, params.family.identity), + }); + while (cache.size > SNAPSHOT_DELTA_CACHE_MAX_ENTRIES) { + const oldest = cache.keys().next().value as string | undefined; + if (oldest === undefined) { + break; + } + cache.delete(oldest); + } +} + +export function clearSnapshotKeysForTab( + ctx: BrowserRouteContext, + profile: string, + targetId: string, +): void { + const cache = cacheByContext.get(ctx); + if (!cache) { + return; + } + for (const [key, entry] of cache) { + if (entry.profile === profile && entry.targetId === targetId) { + cache.delete(key); + } + } +}