mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-26 20:35:39 -06:00
fix(control-ui): restore sandbox media previews (#117689)
* fix(control-ui): restore sandbox media previews * fix(media): preserve anchored history pruning * style: format anchored history pruning * chore: drop release-owned changelog entry * test(ui): cover sandboxed inbound image previews
This commit is contained in:
committed by
GitHub
parent
b2da7aa106
commit
1becbdcb60
@@ -1,3 +1,4 @@
|
||||
import path from "node:path";
|
||||
import { buildInboundMediaNoteProjection } from "../../../auto-reply/media-note.js";
|
||||
import {
|
||||
readPersistedMediaFacts,
|
||||
@@ -100,11 +101,36 @@ function replaceLegacyFactlessMediaText(text: string): string {
|
||||
.replace(LEGACY_INBOUND_MEDIA_URI_PATTERN, PRUNED_HISTORY_MEDIA_REFERENCE_MARKER);
|
||||
}
|
||||
|
||||
function normalizeMarkerIdentity(identity: string): string {
|
||||
return identity.replaceAll("\\", "/");
|
||||
}
|
||||
|
||||
function resolveWorkspaceRelativeMarkerAliases(fact: MediaFact): string[] {
|
||||
if (
|
||||
!fact.path ||
|
||||
!fact.workspaceDir ||
|
||||
!path.isAbsolute(fact.path) ||
|
||||
!path.isAbsolute(fact.workspaceDir)
|
||||
) {
|
||||
return [];
|
||||
}
|
||||
const relativePath = path.relative(fact.workspaceDir, fact.path);
|
||||
if (!relativePath || relativePath.startsWith("..") || path.isAbsolute(relativePath)) {
|
||||
return [];
|
||||
}
|
||||
const normalizedRelativePath = normalizeMarkerIdentity(relativePath);
|
||||
return [normalizedRelativePath, `./${normalizedRelativePath}`];
|
||||
}
|
||||
|
||||
function factOwnsMarkerIdentity(identity: string, media: MediaFact[]): boolean {
|
||||
const normalizedIdentity = identity.replaceAll("\\", "/");
|
||||
return media.some((fact) =>
|
||||
[fact.path, fact.url].some((alias) => alias?.replaceAll("\\", "/") === normalizedIdentity),
|
||||
);
|
||||
const normalizedIdentity = normalizeMarkerIdentity(identity);
|
||||
return media.some((fact) => {
|
||||
// Persistence anchors sandbox paths for browser previews, while existing
|
||||
// prompt marker text remains relative. Derive aliases only from an
|
||||
// explicitly recorded workspace so unrelated absolute facts stay distinct.
|
||||
const aliases = [fact.path, fact.url, ...resolveWorkspaceRelativeMarkerAliases(fact)];
|
||||
return aliases.some((alias) => alias && normalizeMarkerIdentity(alias) === normalizedIdentity);
|
||||
});
|
||||
}
|
||||
|
||||
function extractMediaAttachedIdentity(marker: string): string {
|
||||
|
||||
@@ -49,12 +49,12 @@ export function resolveTranscriptMediaPath(
|
||||
export function normalizeStructuredMediaEntryForTranscript(
|
||||
media: PersistedUserTurnMediaInput,
|
||||
): MediaFactInput {
|
||||
const workspaceDir = normalizeOptionalText(media.workspaceDir);
|
||||
const mediaPath = normalizeOptionalText(media.path);
|
||||
const mediaUrl = normalizeOptionalText(media.url);
|
||||
const kind = normalizeStructuredMediaKind(media.kind);
|
||||
const legacyKind = normalizeOptionalText(media.kind);
|
||||
const messageId = normalizeOptionalText(media.messageId);
|
||||
const workspaceDir = normalizeOptionalText(media.workspaceDir);
|
||||
const contentType =
|
||||
normalizeOptionalText(media.contentType) ??
|
||||
(kind || !legacyKind || !MIME_TYPE_PATTERN.test(legacyKind) ? undefined : legacyKind) ??
|
||||
@@ -65,7 +65,7 @@ export function normalizeStructuredMediaEntryForTranscript(
|
||||
const fileName = normalizeOptionalText(media.fileName);
|
||||
const sizeBytes = normalizeNonNegativeNumber(media.sizeBytes);
|
||||
return {
|
||||
...(mediaPath ? { path: mediaPath } : {}),
|
||||
...(mediaPath ? { path: resolveTranscriptMediaPath(mediaPath, workspaceDir) } : {}),
|
||||
...(mediaUrl ? { url: mediaUrl } : {}),
|
||||
...(contentType ? { contentType } : {}),
|
||||
...(kind ? { kind } : {}),
|
||||
|
||||
@@ -281,6 +281,7 @@ describe("buildPersistedUserTurnMessage media projection", () => {
|
||||
media: [
|
||||
{
|
||||
path: "media/inbound/a.png",
|
||||
url: "https://example.test/original.png",
|
||||
contentType: "image/png",
|
||||
workspaceDir: "/tmp/workspace",
|
||||
},
|
||||
@@ -293,12 +294,24 @@ describe("buildPersistedUserTurnMessage media projection", () => {
|
||||
},
|
||||
expectedMedia: [
|
||||
{
|
||||
path: "media/inbound/a.png",
|
||||
path: path.join("/tmp/workspace", "media/inbound/a.png"),
|
||||
url: "https://example.test/original.png",
|
||||
contentType: "image/png",
|
||||
workspaceDir: "/tmp/workspace",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
name: "unanchored relative attachment",
|
||||
media: [{ path: "media/inbound/unanchored.png", contentType: "image/png" }],
|
||||
expectedLegacy: {
|
||||
MediaPath: "media/inbound/unanchored.png",
|
||||
MediaPaths: ["media/inbound/unanchored.png"],
|
||||
MediaType: "image/png",
|
||||
MediaTypes: ["image/png"],
|
||||
},
|
||||
expectedMedia: [{ path: "media/inbound/unanchored.png", contentType: "image/png" }],
|
||||
},
|
||||
{
|
||||
name: "hydration-suppressed attachment",
|
||||
media: [
|
||||
|
||||
@@ -399,78 +399,101 @@ suite.define(() => {
|
||||
}
|
||||
});
|
||||
|
||||
it("renders a canonical inbound image through the ticketed media route", async () => {
|
||||
const artifactDir = process.env.OPENCLAW_UI_E2E_ARTIFACT_DIR?.trim();
|
||||
const context = await suite.newBrowserContext({
|
||||
locale: "en-US",
|
||||
serviceWorkers: "block",
|
||||
viewport: { height: 900, width: 1280 },
|
||||
});
|
||||
const page = await context.newPage();
|
||||
const requestedMediaUrls: URL[] = [];
|
||||
await page.route("**/__openclaw__/assistant-media?**", async (route) => {
|
||||
const request = route.request();
|
||||
const url = new URL(request.url());
|
||||
requestedMediaUrls.push(url);
|
||||
expect(url.searchParams.get("source")).toBe("media://inbound/telegram-photo.png");
|
||||
if (url.searchParams.get("meta") === "1") {
|
||||
expect(request.headers().authorization).toBe("Bearer e2e-device-token");
|
||||
await route.fulfill({
|
||||
contentType: "application/json",
|
||||
body: JSON.stringify({
|
||||
available: true,
|
||||
mediaTicket: "ticket-inbound",
|
||||
mediaTicketExpiresAt: new Date(Date.now() + 5 * 60_000).toISOString(),
|
||||
}),
|
||||
});
|
||||
return;
|
||||
}
|
||||
expect(url.searchParams.get("mediaTicket")).toBe("ticket-inbound");
|
||||
await route.fulfill({
|
||||
contentType: "image/png",
|
||||
body: Buffer.from(
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Y9Zl1sAAAAASUVORK5CYII=",
|
||||
"base64",
|
||||
),
|
||||
it.each([
|
||||
{
|
||||
name: "canonical inbound",
|
||||
source: "media://inbound/telegram-photo.png",
|
||||
workspaceDir: undefined,
|
||||
screenshotName: "canonical-inbound-image",
|
||||
},
|
||||
{
|
||||
name: "sandbox-staged inbound",
|
||||
source: "/workspace/media/inbound/fabricated-sandbox.png",
|
||||
workspaceDir: "/workspace",
|
||||
screenshotName: "sandbox-inbound-image",
|
||||
},
|
||||
] as const)(
|
||||
"renders a $name image through the ticketed media route",
|
||||
async ({ source, workspaceDir, screenshotName }) => {
|
||||
const artifactDir = process.env.OPENCLAW_UI_E2E_ARTIFACT_DIR?.trim();
|
||||
const context = await suite.newBrowserContext({
|
||||
locale: "en-US",
|
||||
serviceWorkers: "block",
|
||||
viewport: { height: 900, width: 1280 },
|
||||
});
|
||||
});
|
||||
await installMockGateway(page, {
|
||||
historyMessages: [
|
||||
{
|
||||
id: "user-inbound-media-ref",
|
||||
role: "user",
|
||||
content: [{ type: "text", text: "🖼️ Attached image" }],
|
||||
__openclaw: {
|
||||
media: [{ path: "media://inbound/telegram-photo.png", contentType: "image/png" }],
|
||||
},
|
||||
timestamp: Date.now(),
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
try {
|
||||
await page.goto(`${suite.server.baseUrl}chat`);
|
||||
await expect.poll(() => requestedMediaUrls.length, { timeout: 10_000 }).toBe(2);
|
||||
const image = page.locator("img.chat-message-image");
|
||||
await image.waitFor({ state: "visible", timeout: 10_000 });
|
||||
await expect
|
||||
.poll(() =>
|
||||
image.evaluate((element) =>
|
||||
element instanceof HTMLImageElement && element.complete ? element.naturalWidth : 0,
|
||||
const page = await context.newPage();
|
||||
const requestedMediaUrls: URL[] = [];
|
||||
await page.route("**/__openclaw__/assistant-media?**", async (route) => {
|
||||
const request = route.request();
|
||||
const url = new URL(request.url());
|
||||
requestedMediaUrls.push(url);
|
||||
expect(url.searchParams.get("source")).toBe(source);
|
||||
if (url.searchParams.get("meta") === "1") {
|
||||
expect(request.headers().authorization).toBe("Bearer e2e-device-token");
|
||||
await route.fulfill({
|
||||
contentType: "application/json",
|
||||
body: JSON.stringify({
|
||||
available: true,
|
||||
mediaTicket: "ticket-inbound",
|
||||
mediaTicketExpiresAt: new Date(Date.now() + 5 * 60_000).toISOString(),
|
||||
}),
|
||||
});
|
||||
return;
|
||||
}
|
||||
expect(url.searchParams.get("mediaTicket")).toBe("ticket-inbound");
|
||||
expect(request.headers().authorization).toBeUndefined();
|
||||
await route.fulfill({
|
||||
contentType: "image/png",
|
||||
body: Buffer.from(
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Y9Zl1sAAAAASUVORK5CYII=",
|
||||
"base64",
|
||||
),
|
||||
)
|
||||
.toBe(1);
|
||||
if (artifactDir) {
|
||||
await mkdir(artifactDir, { recursive: true });
|
||||
await page.screenshot({
|
||||
fullPage: true,
|
||||
path: `${artifactDir}/canonical-inbound-image.png`,
|
||||
});
|
||||
});
|
||||
await installMockGateway(page, {
|
||||
historyMessages: [
|
||||
{
|
||||
id: "user-inbound-media-ref",
|
||||
role: "user",
|
||||
content: [{ type: "text", text: "🖼️ Attached image" }],
|
||||
__openclaw: {
|
||||
media: [
|
||||
{
|
||||
path: source,
|
||||
contentType: "image/png",
|
||||
...(workspaceDir ? { workspaceDir } : {}),
|
||||
},
|
||||
],
|
||||
},
|
||||
timestamp: Date.now(),
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
try {
|
||||
await page.goto(`${suite.server.baseUrl}chat`);
|
||||
await expect.poll(() => requestedMediaUrls.length, { timeout: 10_000 }).toBe(2);
|
||||
const image = page.locator("img.chat-message-image");
|
||||
await image.waitFor({ state: "visible", timeout: 10_000 });
|
||||
await expect
|
||||
.poll(() =>
|
||||
image.evaluate((element) =>
|
||||
element instanceof HTMLImageElement && element.complete ? element.naturalWidth : 0,
|
||||
),
|
||||
)
|
||||
.toBe(1);
|
||||
if (artifactDir) {
|
||||
await mkdir(artifactDir, { recursive: true });
|
||||
await page.screenshot({
|
||||
fullPage: true,
|
||||
path: `${artifactDir}/${screenshotName}.png`,
|
||||
});
|
||||
}
|
||||
} finally {
|
||||
await suite.closeBrowserContext(context);
|
||||
}
|
||||
} finally {
|
||||
await suite.closeBrowserContext(context);
|
||||
}
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("evicts and refetches managed image Blob URLs after the cache reaches capacity", async () => {
|
||||
const context = await suite.newBrowserContext({
|
||||
|
||||
Reference in New Issue
Block a user