From 1becbdcb60137237017565cb1fd8b79ab0d8efd4 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 1 Aug 2026 19:02:19 -0700 Subject: [PATCH] fix(control-ui): restore sandbox media previews (#117689) * fix(control-ui): restore sandbox media previews * fix(media): preserve anchored history pruning * style: format anchored history pruning * chore: drop release-owned changelog entry * test(ui): cover sandboxed inbound image previews --- .../run/history-image-prune.ts | 34 +++- .../user-turn-transcript.media-normalize.ts | 4 +- .../user-turn-transcript.media.test.ts | 15 +- ui/src/e2e/chat-flow.media-files.e2e.test.ts | 159 ++++++++++-------- 4 files changed, 137 insertions(+), 75 deletions(-) diff --git a/src/agents/embedded-agent-runner/run/history-image-prune.ts b/src/agents/embedded-agent-runner/run/history-image-prune.ts index f18cf56df6be..511c31874225 100644 --- a/src/agents/embedded-agent-runner/run/history-image-prune.ts +++ b/src/agents/embedded-agent-runner/run/history-image-prune.ts @@ -1,3 +1,4 @@ +import path from "node:path"; import { buildInboundMediaNoteProjection } from "../../../auto-reply/media-note.js"; import { readPersistedMediaFacts, @@ -100,11 +101,36 @@ function replaceLegacyFactlessMediaText(text: string): string { .replace(LEGACY_INBOUND_MEDIA_URI_PATTERN, PRUNED_HISTORY_MEDIA_REFERENCE_MARKER); } +function normalizeMarkerIdentity(identity: string): string { + return identity.replaceAll("\\", "/"); +} + +function resolveWorkspaceRelativeMarkerAliases(fact: MediaFact): string[] { + if ( + !fact.path || + !fact.workspaceDir || + !path.isAbsolute(fact.path) || + !path.isAbsolute(fact.workspaceDir) + ) { + return []; + } + const relativePath = path.relative(fact.workspaceDir, fact.path); + if (!relativePath || relativePath.startsWith("..") || path.isAbsolute(relativePath)) { + return []; + } + const normalizedRelativePath = normalizeMarkerIdentity(relativePath); + return [normalizedRelativePath, `./${normalizedRelativePath}`]; +} + function factOwnsMarkerIdentity(identity: string, media: MediaFact[]): boolean { - const normalizedIdentity = identity.replaceAll("\\", "/"); - return media.some((fact) => - [fact.path, fact.url].some((alias) => alias?.replaceAll("\\", "/") === normalizedIdentity), - ); + const normalizedIdentity = normalizeMarkerIdentity(identity); + return media.some((fact) => { + // Persistence anchors sandbox paths for browser previews, while existing + // prompt marker text remains relative. Derive aliases only from an + // explicitly recorded workspace so unrelated absolute facts stay distinct. + const aliases = [fact.path, fact.url, ...resolveWorkspaceRelativeMarkerAliases(fact)]; + return aliases.some((alias) => alias && normalizeMarkerIdentity(alias) === normalizedIdentity); + }); } function extractMediaAttachedIdentity(marker: string): string { diff --git a/src/sessions/user-turn-transcript.media-normalize.ts b/src/sessions/user-turn-transcript.media-normalize.ts index 053d038c82e3..212801735bd9 100644 --- a/src/sessions/user-turn-transcript.media-normalize.ts +++ b/src/sessions/user-turn-transcript.media-normalize.ts @@ -49,12 +49,12 @@ export function resolveTranscriptMediaPath( export function normalizeStructuredMediaEntryForTranscript( media: PersistedUserTurnMediaInput, ): MediaFactInput { + const workspaceDir = normalizeOptionalText(media.workspaceDir); const mediaPath = normalizeOptionalText(media.path); const mediaUrl = normalizeOptionalText(media.url); const kind = normalizeStructuredMediaKind(media.kind); const legacyKind = normalizeOptionalText(media.kind); const messageId = normalizeOptionalText(media.messageId); - const workspaceDir = normalizeOptionalText(media.workspaceDir); const contentType = normalizeOptionalText(media.contentType) ?? (kind || !legacyKind || !MIME_TYPE_PATTERN.test(legacyKind) ? undefined : legacyKind) ?? @@ -65,7 +65,7 @@ export function normalizeStructuredMediaEntryForTranscript( const fileName = normalizeOptionalText(media.fileName); const sizeBytes = normalizeNonNegativeNumber(media.sizeBytes); return { - ...(mediaPath ? { path: mediaPath } : {}), + ...(mediaPath ? { path: resolveTranscriptMediaPath(mediaPath, workspaceDir) } : {}), ...(mediaUrl ? { url: mediaUrl } : {}), ...(contentType ? { contentType } : {}), ...(kind ? { kind } : {}), diff --git a/src/sessions/user-turn-transcript.media.test.ts b/src/sessions/user-turn-transcript.media.test.ts index 0276ae5c387e..30200ced3b81 100644 --- a/src/sessions/user-turn-transcript.media.test.ts +++ b/src/sessions/user-turn-transcript.media.test.ts @@ -281,6 +281,7 @@ describe("buildPersistedUserTurnMessage media projection", () => { media: [ { path: "media/inbound/a.png", + url: "https://example.test/original.png", contentType: "image/png", workspaceDir: "/tmp/workspace", }, @@ -293,12 +294,24 @@ describe("buildPersistedUserTurnMessage media projection", () => { }, expectedMedia: [ { - path: "media/inbound/a.png", + path: path.join("/tmp/workspace", "media/inbound/a.png"), + url: "https://example.test/original.png", contentType: "image/png", workspaceDir: "/tmp/workspace", }, ], }, + { + name: "unanchored relative attachment", + media: [{ path: "media/inbound/unanchored.png", contentType: "image/png" }], + expectedLegacy: { + MediaPath: "media/inbound/unanchored.png", + MediaPaths: ["media/inbound/unanchored.png"], + MediaType: "image/png", + MediaTypes: ["image/png"], + }, + expectedMedia: [{ path: "media/inbound/unanchored.png", contentType: "image/png" }], + }, { name: "hydration-suppressed attachment", media: [ diff --git a/ui/src/e2e/chat-flow.media-files.e2e.test.ts b/ui/src/e2e/chat-flow.media-files.e2e.test.ts index 2a9bafb27bcf..ba19112cabd3 100644 --- a/ui/src/e2e/chat-flow.media-files.e2e.test.ts +++ b/ui/src/e2e/chat-flow.media-files.e2e.test.ts @@ -399,78 +399,101 @@ suite.define(() => { } }); - it("renders a canonical inbound image through the ticketed media route", async () => { - const artifactDir = process.env.OPENCLAW_UI_E2E_ARTIFACT_DIR?.trim(); - const context = await suite.newBrowserContext({ - locale: "en-US", - serviceWorkers: "block", - viewport: { height: 900, width: 1280 }, - }); - const page = await context.newPage(); - const requestedMediaUrls: URL[] = []; - await page.route("**/__openclaw__/assistant-media?**", async (route) => { - const request = route.request(); - const url = new URL(request.url()); - requestedMediaUrls.push(url); - expect(url.searchParams.get("source")).toBe("media://inbound/telegram-photo.png"); - if (url.searchParams.get("meta") === "1") { - expect(request.headers().authorization).toBe("Bearer e2e-device-token"); - await route.fulfill({ - contentType: "application/json", - body: JSON.stringify({ - available: true, - mediaTicket: "ticket-inbound", - mediaTicketExpiresAt: new Date(Date.now() + 5 * 60_000).toISOString(), - }), - }); - return; - } - expect(url.searchParams.get("mediaTicket")).toBe("ticket-inbound"); - await route.fulfill({ - contentType: "image/png", - body: Buffer.from( - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Y9Zl1sAAAAASUVORK5CYII=", - "base64", - ), + it.each([ + { + name: "canonical inbound", + source: "media://inbound/telegram-photo.png", + workspaceDir: undefined, + screenshotName: "canonical-inbound-image", + }, + { + name: "sandbox-staged inbound", + source: "/workspace/media/inbound/fabricated-sandbox.png", + workspaceDir: "/workspace", + screenshotName: "sandbox-inbound-image", + }, + ] as const)( + "renders a $name image through the ticketed media route", + async ({ source, workspaceDir, screenshotName }) => { + const artifactDir = process.env.OPENCLAW_UI_E2E_ARTIFACT_DIR?.trim(); + const context = await suite.newBrowserContext({ + locale: "en-US", + serviceWorkers: "block", + viewport: { height: 900, width: 1280 }, }); - }); - await installMockGateway(page, { - historyMessages: [ - { - id: "user-inbound-media-ref", - role: "user", - content: [{ type: "text", text: "🖼️ Attached image" }], - __openclaw: { - media: [{ path: "media://inbound/telegram-photo.png", contentType: "image/png" }], - }, - timestamp: Date.now(), - }, - ], - }); - - try { - await page.goto(`${suite.server.baseUrl}chat`); - await expect.poll(() => requestedMediaUrls.length, { timeout: 10_000 }).toBe(2); - const image = page.locator("img.chat-message-image"); - await image.waitFor({ state: "visible", timeout: 10_000 }); - await expect - .poll(() => - image.evaluate((element) => - element instanceof HTMLImageElement && element.complete ? element.naturalWidth : 0, + const page = await context.newPage(); + const requestedMediaUrls: URL[] = []; + await page.route("**/__openclaw__/assistant-media?**", async (route) => { + const request = route.request(); + const url = new URL(request.url()); + requestedMediaUrls.push(url); + expect(url.searchParams.get("source")).toBe(source); + if (url.searchParams.get("meta") === "1") { + expect(request.headers().authorization).toBe("Bearer e2e-device-token"); + await route.fulfill({ + contentType: "application/json", + body: JSON.stringify({ + available: true, + mediaTicket: "ticket-inbound", + mediaTicketExpiresAt: new Date(Date.now() + 5 * 60_000).toISOString(), + }), + }); + return; + } + expect(url.searchParams.get("mediaTicket")).toBe("ticket-inbound"); + expect(request.headers().authorization).toBeUndefined(); + await route.fulfill({ + contentType: "image/png", + body: Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Y9Zl1sAAAAASUVORK5CYII=", + "base64", ), - ) - .toBe(1); - if (artifactDir) { - await mkdir(artifactDir, { recursive: true }); - await page.screenshot({ - fullPage: true, - path: `${artifactDir}/canonical-inbound-image.png`, }); + }); + await installMockGateway(page, { + historyMessages: [ + { + id: "user-inbound-media-ref", + role: "user", + content: [{ type: "text", text: "🖼️ Attached image" }], + __openclaw: { + media: [ + { + path: source, + contentType: "image/png", + ...(workspaceDir ? { workspaceDir } : {}), + }, + ], + }, + timestamp: Date.now(), + }, + ], + }); + + try { + await page.goto(`${suite.server.baseUrl}chat`); + await expect.poll(() => requestedMediaUrls.length, { timeout: 10_000 }).toBe(2); + const image = page.locator("img.chat-message-image"); + await image.waitFor({ state: "visible", timeout: 10_000 }); + await expect + .poll(() => + image.evaluate((element) => + element instanceof HTMLImageElement && element.complete ? element.naturalWidth : 0, + ), + ) + .toBe(1); + if (artifactDir) { + await mkdir(artifactDir, { recursive: true }); + await page.screenshot({ + fullPage: true, + path: `${artifactDir}/${screenshotName}.png`, + }); + } + } finally { + await suite.closeBrowserContext(context); } - } finally { - await suite.closeBrowserContext(context); - } - }); + }, + ); it("evicts and refetches managed image Blob URLs after the cache reaches capacity", async () => { const context = await suite.newBrowserContext({