mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-27 12:56:01 -06:00
fix(browser): preserve agent identity for screenshot vision (#121941)
Fixes #121919. Preserve screenshot session ownership through the focused agent-scope SDK before media runtime resolves provider credentials. Co-authored-by: Peter Steinberger <steipete@gmail.com>
This commit is contained in:
@@ -261,6 +261,7 @@ describe("browser plugin", () => {
|
||||
await tool.execute("call-1", { action: "status" });
|
||||
expect(runtimeApiMocks.createBrowserTool).toHaveBeenCalledWith({
|
||||
agentSessionKey: "agent:main:webchat:direct:123",
|
||||
agentId: "main",
|
||||
agentDir: "/tmp/agent",
|
||||
workspaceDir: "/tmp/workspace",
|
||||
activeModel: { provider: "openai", model: "gpt-5.5" },
|
||||
|
||||
@@ -77,6 +77,7 @@ function createLazyBrowserTool(
|
||||
sandboxBridgeUrl?: string;
|
||||
allowHostControl?: boolean;
|
||||
agentSessionKey?: string;
|
||||
agentId?: string;
|
||||
agentDir?: string;
|
||||
workspaceDir?: string;
|
||||
activeModel?: {
|
||||
@@ -138,6 +139,7 @@ function createBrowserToolOptions(ctx: OpenClawPluginToolContext): {
|
||||
sandboxBridgeUrl?: string;
|
||||
allowHostControl?: boolean;
|
||||
agentSessionKey?: string;
|
||||
agentId?: string;
|
||||
agentDir?: string;
|
||||
workspaceDir?: string;
|
||||
activeModel?: {
|
||||
@@ -159,6 +161,7 @@ function createBrowserToolOptions(ctx: OpenClawPluginToolContext): {
|
||||
? { allowHostControl: ctx.browser.allowHostControl }
|
||||
: {}),
|
||||
...(ctx.sessionKey ? { agentSessionKey: ctx.sessionKey } : {}),
|
||||
...(ctx.agentId ? { agentId: ctx.agentId } : {}),
|
||||
...(ctx.agentDir ? { agentDir: ctx.agentDir } : {}),
|
||||
...(ctx.workspaceDir ? { workspaceDir: ctx.workspaceDir } : {}),
|
||||
...(ctx.activeModel?.provider || ctx.activeModel?.modelId
|
||||
|
||||
@@ -364,6 +364,7 @@ export function createBrowserTool(opts?: {
|
||||
sandboxBridgeUrl?: string;
|
||||
allowHostControl?: boolean;
|
||||
agentSessionKey?: string;
|
||||
agentId?: string;
|
||||
agentDir?: string;
|
||||
workspaceDir?: string;
|
||||
activeModel?: {
|
||||
@@ -835,6 +836,7 @@ export function createBrowserTool(opts?: {
|
||||
cfg: screenshotCfg,
|
||||
filePath: screenshotPath,
|
||||
agentDir: opts?.agentDir,
|
||||
agentId: opts?.agentId,
|
||||
workspaceDir: opts?.workspaceDir,
|
||||
activeModel: opts?.activeModel,
|
||||
mediaScope: opts?.mediaScope,
|
||||
|
||||
@@ -92,6 +92,48 @@ describe("describeBrowserScreenshot", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ name: "session-owned default agent", agentId: undefined, sessionAgentId: "main" },
|
||||
{ name: "explicit matching worker agent", agentId: "worker", sessionAgentId: "worker" },
|
||||
])(
|
||||
"passes the $name identity to image understanding when its directory is absent",
|
||||
async (testCase) => {
|
||||
const describeEntry = vi.fn().mockResolvedValue({ text: "A dashboard." });
|
||||
|
||||
await describeBrowserScreenshot(
|
||||
{
|
||||
cfg: {},
|
||||
filePath: "/tmp/screenshot.png",
|
||||
...(testCase.agentId ? { agentId: testCase.agentId } : {}),
|
||||
mediaScope: { sessionKey: `agent:${testCase.sessionAgentId}:webchat:direct:123` },
|
||||
},
|
||||
makeDeps(describeEntry),
|
||||
);
|
||||
|
||||
expect(describeEntry).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ agentId: testCase.sessionAgentId, agentDir: undefined }),
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it("rejects an agent identity that conflicts with its session before image understanding", async () => {
|
||||
const describeEntry = vi.fn().mockResolvedValue({ text: "A dashboard." });
|
||||
|
||||
await expect(
|
||||
describeBrowserScreenshot(
|
||||
{
|
||||
cfg: {},
|
||||
filePath: "/tmp/screenshot.png",
|
||||
agentId: "worker",
|
||||
mediaScope: { sessionKey: "agent:main:webchat:direct:123" },
|
||||
},
|
||||
makeDeps(describeEntry),
|
||||
),
|
||||
).rejects.toThrow(/belongs to "main", not "worker"/);
|
||||
|
||||
expect(describeEntry).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("resizes screenshots before image understanding when image sanitization is configured", async () => {
|
||||
const describeResult = vi.fn().mockResolvedValue({ text: "Small screenshot." });
|
||||
const normalizeBrowserScreenshot = vi.fn(async () => ({
|
||||
|
||||
@@ -90,10 +90,18 @@ export async function describeBrowserScreenshot(
|
||||
deps: BrowserScreenshotDescriptionDeps,
|
||||
): Promise<BrowserScreenshotDescriptionResult | null> {
|
||||
const filePath = await resolveImageUnderstandingFilePath(ctx, deps);
|
||||
const agentId = ctx.agentDir
|
||||
? undefined
|
||||
: (await import("openclaw/plugin-sdk/agent-scope-runtime")).resolveSessionAgentId({
|
||||
agentId: ctx.agentId,
|
||||
sessionKey: ctx.mediaScope?.sessionKey,
|
||||
config: ctx.cfg,
|
||||
});
|
||||
const described = await deps.describeImageFile({
|
||||
filePath,
|
||||
cfg: ctx.cfg,
|
||||
prompt: DEFAULT_BROWSER_SCREENSHOT_DESCRIPTION_PROMPT,
|
||||
...(agentId ? { agentId } : {}),
|
||||
agentDir: ctx.agentDir,
|
||||
workspaceDir: ctx.workspaceDir,
|
||||
activeModel: normalizeActiveModel(ctx.activeModel),
|
||||
|
||||
Reference in New Issue
Block a user