fix(browser): preserve agent identity for screenshot vision (#121941)

Fixes #121919. Preserve screenshot session ownership through the focused agent-scope SDK before media runtime resolves provider credentials.

Co-authored-by: Peter Steinberger <steipete@gmail.com>
This commit is contained in:
nissl24
2026-08-25 10:04:55 +02:00
committed by GitHub
parent 1901ddaa81
commit 8f243929e4
5 changed files with 56 additions and 0 deletions
+1
View File
@@ -261,6 +261,7 @@ describe("browser plugin", () => {
await tool.execute("call-1", { action: "status" });
expect(runtimeApiMocks.createBrowserTool).toHaveBeenCalledWith({
agentSessionKey: "agent:main:webchat:direct:123",
agentId: "main",
agentDir: "/tmp/agent",
workspaceDir: "/tmp/workspace",
activeModel: { provider: "openai", model: "gpt-5.5" },
@@ -77,6 +77,7 @@ function createLazyBrowserTool(
sandboxBridgeUrl?: string;
allowHostControl?: boolean;
agentSessionKey?: string;
agentId?: string;
agentDir?: string;
workspaceDir?: string;
activeModel?: {
@@ -138,6 +139,7 @@ function createBrowserToolOptions(ctx: OpenClawPluginToolContext): {
sandboxBridgeUrl?: string;
allowHostControl?: boolean;
agentSessionKey?: string;
agentId?: string;
agentDir?: string;
workspaceDir?: string;
activeModel?: {
@@ -159,6 +161,7 @@ function createBrowserToolOptions(ctx: OpenClawPluginToolContext): {
? { allowHostControl: ctx.browser.allowHostControl }
: {}),
...(ctx.sessionKey ? { agentSessionKey: ctx.sessionKey } : {}),
...(ctx.agentId ? { agentId: ctx.agentId } : {}),
...(ctx.agentDir ? { agentDir: ctx.agentDir } : {}),
...(ctx.workspaceDir ? { workspaceDir: ctx.workspaceDir } : {}),
...(ctx.activeModel?.provider || ctx.activeModel?.modelId
+2
View File
@@ -364,6 +364,7 @@ export function createBrowserTool(opts?: {
sandboxBridgeUrl?: string;
allowHostControl?: boolean;
agentSessionKey?: string;
agentId?: string;
agentDir?: string;
workspaceDir?: string;
activeModel?: {
@@ -835,6 +836,7 @@ export function createBrowserTool(opts?: {
cfg: screenshotCfg,
filePath: screenshotPath,
agentDir: opts?.agentDir,
agentId: opts?.agentId,
workspaceDir: opts?.workspaceDir,
activeModel: opts?.activeModel,
mediaScope: opts?.mediaScope,
@@ -92,6 +92,48 @@ describe("describeBrowserScreenshot", () => {
});
});
it.each([
{ name: "session-owned default agent", agentId: undefined, sessionAgentId: "main" },
{ name: "explicit matching worker agent", agentId: "worker", sessionAgentId: "worker" },
])(
"passes the $name identity to image understanding when its directory is absent",
async (testCase) => {
const describeEntry = vi.fn().mockResolvedValue({ text: "A dashboard." });
await describeBrowserScreenshot(
{
cfg: {},
filePath: "/tmp/screenshot.png",
...(testCase.agentId ? { agentId: testCase.agentId } : {}),
mediaScope: { sessionKey: `agent:${testCase.sessionAgentId}:webchat:direct:123` },
},
makeDeps(describeEntry),
);
expect(describeEntry).toHaveBeenCalledWith(
expect.objectContaining({ agentId: testCase.sessionAgentId, agentDir: undefined }),
);
},
);
it("rejects an agent identity that conflicts with its session before image understanding", async () => {
const describeEntry = vi.fn().mockResolvedValue({ text: "A dashboard." });
await expect(
describeBrowserScreenshot(
{
cfg: {},
filePath: "/tmp/screenshot.png",
agentId: "worker",
mediaScope: { sessionKey: "agent:main:webchat:direct:123" },
},
makeDeps(describeEntry),
),
).rejects.toThrow(/belongs to "main", not "worker"/);
expect(describeEntry).not.toHaveBeenCalled();
});
it("resizes screenshots before image understanding when image sanitization is configured", async () => {
const describeResult = vi.fn().mockResolvedValue({ text: "Small screenshot." });
const normalizeBrowserScreenshot = vi.fn(async () => ({
+8
View File
@@ -90,10 +90,18 @@ export async function describeBrowserScreenshot(
deps: BrowserScreenshotDescriptionDeps,
): Promise<BrowserScreenshotDescriptionResult | null> {
const filePath = await resolveImageUnderstandingFilePath(ctx, deps);
const agentId = ctx.agentDir
? undefined
: (await import("openclaw/plugin-sdk/agent-scope-runtime")).resolveSessionAgentId({
agentId: ctx.agentId,
sessionKey: ctx.mediaScope?.sessionKey,
config: ctx.cfg,
});
const described = await deps.describeImageFile({
filePath,
cfg: ctx.cfg,
prompt: DEFAULT_BROWSER_SCREENSHOT_DESCRIPTION_PROMPT,
...(agentId ? { agentId } : {}),
agentDir: ctx.agentDir,
workspaceDir: ctx.workspaceDir,
activeModel: normalizeActiveModel(ctx.activeModel),