From 8f243929e42fa7fe216d69eb4a8d598bdb8afe76 Mon Sep 17 00:00:00 2001 From: nissl24 Date: Tue, 25 Aug 2026 10:04:55 +0200 Subject: [PATCH] fix(browser): preserve agent identity for screenshot vision (#121941) Fixes #121919. Preserve screenshot session ownership through the focused agent-scope SDK before media runtime resolves provider credentials. Co-authored-by: Peter Steinberger --- extensions/browser/index.test.ts | 1 + extensions/browser/plugin-registration.ts | 3 ++ extensions/browser/src/browser-tool.ts | 2 + extensions/browser/src/browser/vision.test.ts | 42 +++++++++++++++++++ extensions/browser/src/browser/vision.ts | 8 ++++ 5 files changed, 56 insertions(+) diff --git a/extensions/browser/index.test.ts b/extensions/browser/index.test.ts index db3642c8cbc9..704a762802c9 100644 --- a/extensions/browser/index.test.ts +++ b/extensions/browser/index.test.ts @@ -261,6 +261,7 @@ describe("browser plugin", () => { await tool.execute("call-1", { action: "status" }); expect(runtimeApiMocks.createBrowserTool).toHaveBeenCalledWith({ agentSessionKey: "agent:main:webchat:direct:123", + agentId: "main", agentDir: "/tmp/agent", workspaceDir: "/tmp/workspace", activeModel: { provider: "openai", model: "gpt-5.5" }, diff --git a/extensions/browser/plugin-registration.ts b/extensions/browser/plugin-registration.ts index 35e77fd4d912..39e0245cc245 100644 --- a/extensions/browser/plugin-registration.ts +++ b/extensions/browser/plugin-registration.ts @@ -77,6 +77,7 @@ function createLazyBrowserTool( sandboxBridgeUrl?: string; allowHostControl?: boolean; agentSessionKey?: string; + agentId?: string; agentDir?: string; workspaceDir?: string; activeModel?: { @@ -138,6 +139,7 @@ function createBrowserToolOptions(ctx: OpenClawPluginToolContext): { sandboxBridgeUrl?: string; allowHostControl?: boolean; agentSessionKey?: string; + agentId?: string; agentDir?: string; workspaceDir?: string; activeModel?: { @@ -159,6 +161,7 @@ function createBrowserToolOptions(ctx: OpenClawPluginToolContext): { ? { allowHostControl: ctx.browser.allowHostControl } : {}), ...(ctx.sessionKey ? { agentSessionKey: ctx.sessionKey } : {}), + ...(ctx.agentId ? { agentId: ctx.agentId } : {}), ...(ctx.agentDir ? { agentDir: ctx.agentDir } : {}), ...(ctx.workspaceDir ? { workspaceDir: ctx.workspaceDir } : {}), ...(ctx.activeModel?.provider || ctx.activeModel?.modelId diff --git a/extensions/browser/src/browser-tool.ts b/extensions/browser/src/browser-tool.ts index 0b9306a3fa3c..dbac56cb12b8 100644 --- a/extensions/browser/src/browser-tool.ts +++ b/extensions/browser/src/browser-tool.ts @@ -364,6 +364,7 @@ export function createBrowserTool(opts?: { sandboxBridgeUrl?: string; allowHostControl?: boolean; agentSessionKey?: string; + agentId?: string; agentDir?: string; workspaceDir?: string; activeModel?: { @@ -835,6 +836,7 @@ export function createBrowserTool(opts?: { cfg: screenshotCfg, filePath: screenshotPath, agentDir: opts?.agentDir, + agentId: opts?.agentId, workspaceDir: opts?.workspaceDir, activeModel: opts?.activeModel, mediaScope: opts?.mediaScope, diff --git a/extensions/browser/src/browser/vision.test.ts b/extensions/browser/src/browser/vision.test.ts index 9222b0258deb..f82b70de6789 100644 --- a/extensions/browser/src/browser/vision.test.ts +++ b/extensions/browser/src/browser/vision.test.ts @@ -92,6 +92,48 @@ describe("describeBrowserScreenshot", () => { }); }); + it.each([ + { name: "session-owned default agent", agentId: undefined, sessionAgentId: "main" }, + { name: "explicit matching worker agent", agentId: "worker", sessionAgentId: "worker" }, + ])( + "passes the $name identity to image understanding when its directory is absent", + async (testCase) => { + const describeEntry = vi.fn().mockResolvedValue({ text: "A dashboard." }); + + await describeBrowserScreenshot( + { + cfg: {}, + filePath: "/tmp/screenshot.png", + ...(testCase.agentId ? { agentId: testCase.agentId } : {}), + mediaScope: { sessionKey: `agent:${testCase.sessionAgentId}:webchat:direct:123` }, + }, + makeDeps(describeEntry), + ); + + expect(describeEntry).toHaveBeenCalledWith( + expect.objectContaining({ agentId: testCase.sessionAgentId, agentDir: undefined }), + ); + }, + ); + + it("rejects an agent identity that conflicts with its session before image understanding", async () => { + const describeEntry = vi.fn().mockResolvedValue({ text: "A dashboard." }); + + await expect( + describeBrowserScreenshot( + { + cfg: {}, + filePath: "/tmp/screenshot.png", + agentId: "worker", + mediaScope: { sessionKey: "agent:main:webchat:direct:123" }, + }, + makeDeps(describeEntry), + ), + ).rejects.toThrow(/belongs to "main", not "worker"/); + + expect(describeEntry).not.toHaveBeenCalled(); + }); + it("resizes screenshots before image understanding when image sanitization is configured", async () => { const describeResult = vi.fn().mockResolvedValue({ text: "Small screenshot." }); const normalizeBrowserScreenshot = vi.fn(async () => ({ diff --git a/extensions/browser/src/browser/vision.ts b/extensions/browser/src/browser/vision.ts index 1c321abf7fa3..f85819e86eff 100644 --- a/extensions/browser/src/browser/vision.ts +++ b/extensions/browser/src/browser/vision.ts @@ -90,10 +90,18 @@ export async function describeBrowserScreenshot( deps: BrowserScreenshotDescriptionDeps, ): Promise { const filePath = await resolveImageUnderstandingFilePath(ctx, deps); + const agentId = ctx.agentDir + ? undefined + : (await import("openclaw/plugin-sdk/agent-scope-runtime")).resolveSessionAgentId({ + agentId: ctx.agentId, + sessionKey: ctx.mediaScope?.sessionKey, + config: ctx.cfg, + }); const described = await deps.describeImageFile({ filePath, cfg: ctx.cfg, prompt: DEFAULT_BROWSER_SCREENSHOT_DESCRIPTION_PROMPT, + ...(agentId ? { agentId } : {}), agentDir: ctx.agentDir, workspaceDir: ctx.workspaceDir, activeModel: normalizeActiveModel(ctx.activeModel),