fix(control-ui): restore sandbox media previews (#117689)

* fix(control-ui): restore sandbox media previews

* fix(media): preserve anchored history pruning

* style: format anchored history pruning

* chore: drop release-owned changelog entry

* test(ui): cover sandboxed inbound image previews
This commit is contained in:
Peter Steinberger
2026-08-01 19:02:19 -07:00
committed by GitHub
parent b2da7aa106
commit 1becbdcb60
4 changed files with 137 additions and 75 deletions
@@ -1,3 +1,4 @@
import path from "node:path";
import { buildInboundMediaNoteProjection } from "../../../auto-reply/media-note.js";
import {
readPersistedMediaFacts,
@@ -100,11 +101,36 @@ function replaceLegacyFactlessMediaText(text: string): string {
.replace(LEGACY_INBOUND_MEDIA_URI_PATTERN, PRUNED_HISTORY_MEDIA_REFERENCE_MARKER);
}
function normalizeMarkerIdentity(identity: string): string {
return identity.replaceAll("\\", "/");
}
function resolveWorkspaceRelativeMarkerAliases(fact: MediaFact): string[] {
if (
!fact.path ||
!fact.workspaceDir ||
!path.isAbsolute(fact.path) ||
!path.isAbsolute(fact.workspaceDir)
) {
return [];
}
const relativePath = path.relative(fact.workspaceDir, fact.path);
if (!relativePath || relativePath.startsWith("..") || path.isAbsolute(relativePath)) {
return [];
}
const normalizedRelativePath = normalizeMarkerIdentity(relativePath);
return [normalizedRelativePath, `./${normalizedRelativePath}`];
}
function factOwnsMarkerIdentity(identity: string, media: MediaFact[]): boolean {
const normalizedIdentity = identity.replaceAll("\\", "/");
return media.some((fact) =>
[fact.path, fact.url].some((alias) => alias?.replaceAll("\\", "/") === normalizedIdentity),
);
const normalizedIdentity = normalizeMarkerIdentity(identity);
return media.some((fact) => {
// Persistence anchors sandbox paths for browser previews, while existing
// prompt marker text remains relative. Derive aliases only from an
// explicitly recorded workspace so unrelated absolute facts stay distinct.
const aliases = [fact.path, fact.url, ...resolveWorkspaceRelativeMarkerAliases(fact)];
return aliases.some((alias) => alias && normalizeMarkerIdentity(alias) === normalizedIdentity);
});
}
function extractMediaAttachedIdentity(marker: string): string {
@@ -49,12 +49,12 @@ export function resolveTranscriptMediaPath(
export function normalizeStructuredMediaEntryForTranscript(
media: PersistedUserTurnMediaInput,
): MediaFactInput {
const workspaceDir = normalizeOptionalText(media.workspaceDir);
const mediaPath = normalizeOptionalText(media.path);
const mediaUrl = normalizeOptionalText(media.url);
const kind = normalizeStructuredMediaKind(media.kind);
const legacyKind = normalizeOptionalText(media.kind);
const messageId = normalizeOptionalText(media.messageId);
const workspaceDir = normalizeOptionalText(media.workspaceDir);
const contentType =
normalizeOptionalText(media.contentType) ??
(kind || !legacyKind || !MIME_TYPE_PATTERN.test(legacyKind) ? undefined : legacyKind) ??
@@ -65,7 +65,7 @@ export function normalizeStructuredMediaEntryForTranscript(
const fileName = normalizeOptionalText(media.fileName);
const sizeBytes = normalizeNonNegativeNumber(media.sizeBytes);
return {
...(mediaPath ? { path: mediaPath } : {}),
...(mediaPath ? { path: resolveTranscriptMediaPath(mediaPath, workspaceDir) } : {}),
...(mediaUrl ? { url: mediaUrl } : {}),
...(contentType ? { contentType } : {}),
...(kind ? { kind } : {}),
@@ -281,6 +281,7 @@ describe("buildPersistedUserTurnMessage media projection", () => {
media: [
{
path: "media/inbound/a.png",
url: "https://example.test/original.png",
contentType: "image/png",
workspaceDir: "/tmp/workspace",
},
@@ -293,12 +294,24 @@ describe("buildPersistedUserTurnMessage media projection", () => {
},
expectedMedia: [
{
path: "media/inbound/a.png",
path: path.join("/tmp/workspace", "media/inbound/a.png"),
url: "https://example.test/original.png",
contentType: "image/png",
workspaceDir: "/tmp/workspace",
},
],
},
{
name: "unanchored relative attachment",
media: [{ path: "media/inbound/unanchored.png", contentType: "image/png" }],
expectedLegacy: {
MediaPath: "media/inbound/unanchored.png",
MediaPaths: ["media/inbound/unanchored.png"],
MediaType: "image/png",
MediaTypes: ["image/png"],
},
expectedMedia: [{ path: "media/inbound/unanchored.png", contentType: "image/png" }],
},
{
name: "hydration-suppressed attachment",
media: [
+91 -68
View File
@@ -399,78 +399,101 @@ suite.define(() => {
}
});
it("renders a canonical inbound image through the ticketed media route", async () => {
const artifactDir = process.env.OPENCLAW_UI_E2E_ARTIFACT_DIR?.trim();
const context = await suite.newBrowserContext({
locale: "en-US",
serviceWorkers: "block",
viewport: { height: 900, width: 1280 },
});
const page = await context.newPage();
const requestedMediaUrls: URL[] = [];
await page.route("**/__openclaw__/assistant-media?**", async (route) => {
const request = route.request();
const url = new URL(request.url());
requestedMediaUrls.push(url);
expect(url.searchParams.get("source")).toBe("media://inbound/telegram-photo.png");
if (url.searchParams.get("meta") === "1") {
expect(request.headers().authorization).toBe("Bearer e2e-device-token");
await route.fulfill({
contentType: "application/json",
body: JSON.stringify({
available: true,
mediaTicket: "ticket-inbound",
mediaTicketExpiresAt: new Date(Date.now() + 5 * 60_000).toISOString(),
}),
});
return;
}
expect(url.searchParams.get("mediaTicket")).toBe("ticket-inbound");
await route.fulfill({
contentType: "image/png",
body: Buffer.from(
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Y9Zl1sAAAAASUVORK5CYII=",
"base64",
),
it.each([
{
name: "canonical inbound",
source: "media://inbound/telegram-photo.png",
workspaceDir: undefined,
screenshotName: "canonical-inbound-image",
},
{
name: "sandbox-staged inbound",
source: "/workspace/media/inbound/fabricated-sandbox.png",
workspaceDir: "/workspace",
screenshotName: "sandbox-inbound-image",
},
] as const)(
"renders a $name image through the ticketed media route",
async ({ source, workspaceDir, screenshotName }) => {
const artifactDir = process.env.OPENCLAW_UI_E2E_ARTIFACT_DIR?.trim();
const context = await suite.newBrowserContext({
locale: "en-US",
serviceWorkers: "block",
viewport: { height: 900, width: 1280 },
});
});
await installMockGateway(page, {
historyMessages: [
{
id: "user-inbound-media-ref",
role: "user",
content: [{ type: "text", text: "🖼️ Attached image" }],
__openclaw: {
media: [{ path: "media://inbound/telegram-photo.png", contentType: "image/png" }],
},
timestamp: Date.now(),
},
],
});
try {
await page.goto(`${suite.server.baseUrl}chat`);
await expect.poll(() => requestedMediaUrls.length, { timeout: 10_000 }).toBe(2);
const image = page.locator("img.chat-message-image");
await image.waitFor({ state: "visible", timeout: 10_000 });
await expect
.poll(() =>
image.evaluate((element) =>
element instanceof HTMLImageElement && element.complete ? element.naturalWidth : 0,
const page = await context.newPage();
const requestedMediaUrls: URL[] = [];
await page.route("**/__openclaw__/assistant-media?**", async (route) => {
const request = route.request();
const url = new URL(request.url());
requestedMediaUrls.push(url);
expect(url.searchParams.get("source")).toBe(source);
if (url.searchParams.get("meta") === "1") {
expect(request.headers().authorization).toBe("Bearer e2e-device-token");
await route.fulfill({
contentType: "application/json",
body: JSON.stringify({
available: true,
mediaTicket: "ticket-inbound",
mediaTicketExpiresAt: new Date(Date.now() + 5 * 60_000).toISOString(),
}),
});
return;
}
expect(url.searchParams.get("mediaTicket")).toBe("ticket-inbound");
expect(request.headers().authorization).toBeUndefined();
await route.fulfill({
contentType: "image/png",
body: Buffer.from(
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Y9Zl1sAAAAASUVORK5CYII=",
"base64",
),
)
.toBe(1);
if (artifactDir) {
await mkdir(artifactDir, { recursive: true });
await page.screenshot({
fullPage: true,
path: `${artifactDir}/canonical-inbound-image.png`,
});
});
await installMockGateway(page, {
historyMessages: [
{
id: "user-inbound-media-ref",
role: "user",
content: [{ type: "text", text: "🖼️ Attached image" }],
__openclaw: {
media: [
{
path: source,
contentType: "image/png",
...(workspaceDir ? { workspaceDir } : {}),
},
],
},
timestamp: Date.now(),
},
],
});
try {
await page.goto(`${suite.server.baseUrl}chat`);
await expect.poll(() => requestedMediaUrls.length, { timeout: 10_000 }).toBe(2);
const image = page.locator("img.chat-message-image");
await image.waitFor({ state: "visible", timeout: 10_000 });
await expect
.poll(() =>
image.evaluate((element) =>
element instanceof HTMLImageElement && element.complete ? element.naturalWidth : 0,
),
)
.toBe(1);
if (artifactDir) {
await mkdir(artifactDir, { recursive: true });
await page.screenshot({
fullPage: true,
path: `${artifactDir}/${screenshotName}.png`,
});
}
} finally {
await suite.closeBrowserContext(context);
}
} finally {
await suite.closeBrowserContext(context);
}
});
},
);
it("evicts and refetches managed image Blob URLs after the cache reaches capacity", async () => {
const context = await suite.newBrowserContext({