diff --git a/src/agents/openclaw-tools.sessions.test.ts b/src/agents/openclaw-tools.sessions.test.ts index 035b850778ee..edc1f7c82007 100644 --- a/src/agents/openclaw-tools.sessions.test.ts +++ b/src/agents/openclaw-tools.sessions.test.ts @@ -737,7 +737,7 @@ describe("sessions tools", () => { ); }); - it("sessions_history caps oversized payloads and strips heavy fields", async () => { + it("sessions_history caps oversized payloads and strips tool-owned heavy fields", async () => { const oversized = Array.from({ length: 80 }, (_, idx) => ({ role: "assistant", content: [ @@ -748,14 +748,6 @@ describe("sessions tools", () => { { type: "thinking", thinking: "y".repeat(7000), - thinkingSignature: "sig".repeat(4000), - openclawReasoningReplay: { - v: 1, - source: "openai-responses", - provider: "openai", - api: "openai-chatgpt-responses", - model: "gpt-5.5", - }, }, ], details: { @@ -765,10 +757,6 @@ describe("sessions tools", () => { input: 1, output: 1, }, - providerReplay: { - type: "openai-responses-compaction", - data: "opaque-sessions-history-compaction", - }, })); callGatewayMock.mockImplementation(async (opts: unknown) => { const request = opts as { method?: string }; @@ -795,7 +783,7 @@ describe("sessions tools", () => { expect(details.truncated).toBe(true); expect(details.droppedMessages).toBe(true); expect(details.contentTruncated).toBe(true); - expect(details.contentRedacted).toBe(true); + expect(details.contentRedacted).toBe(false); expect(typeof details.bytes).toBe("number"); expect((details.bytes ?? 0) <= 80 * 1024).toBe(true); expect(details.messages && details.messages.length > 0).toBe(true); @@ -804,26 +792,20 @@ describe("sessions tools", () => { | { details?: unknown; usage?: unknown; - providerReplay?: unknown; content?: Array<{ type?: string; text?: string; thinking?: string; - thinkingSignature?: string; - openclawReasoningReplay?: unknown; }>; } | undefined; expect(first?.details).toBeUndefined(); expect(first?.usage).toBeUndefined(); - expect(first?.providerReplay).toBeUndefined(); - expect(JSON.stringify(details.messages)).not.toContain("opaque-sessions-history-compaction"); const textBlock = first?.content?.find((block) => block.type === "text"); expect(typeof textBlock?.text).toBe("string"); expect((textBlock?.text ?? "").length <= 4015).toBe(true); const thinkingBlock = first?.content?.find((block) => block.type === "thinking"); - expect(thinkingBlock?.thinkingSignature).toBeUndefined(); - expect(thinkingBlock?.openclawReasoningReplay).toBeUndefined(); + expect((thinkingBlock?.thinking ?? "").length <= 4015).toBe(true); }); it("sessions_history enforces a hard byte cap even when a single message is huge", async () => { diff --git a/src/agents/tools/sessions-history-tool.test.ts b/src/agents/tools/sessions-history-tool.test.ts index 1189dabe56b0..7206a5bcc632 100644 --- a/src/agents/tools/sessions-history-tool.test.ts +++ b/src/agents/tools/sessions-history-tool.test.ts @@ -175,42 +175,6 @@ describe("sessions_history redaction", () => { expect((result.details as { contentRedacted?: unknown }).contentRedacted).toBe(true); }); - it.each([ - { - name: "reports decoded bytes for raw image data", - content: [ - { - type: "image", - data: Buffer.from([0, 1, 2, 3, 4]).toString("base64"), - mimeType: "image/png", - }, - ], - expectedBytes: 5, - }, - { - name: "replaces stale bytes for empty image data", - content: [{ type: "image", data: "", mimeType: "image/png", bytes: 999 }], - expectedBytes: 0, - }, - { - name: "preserves existing bytes when image data is already omitted", - content: [{ type: "image", mimeType: "image/png", bytes: 37, omitted: true }], - expectedBytes: 37, - }, - ])("$name", async ({ content, expectedBytes }) => { - const tool = createHistoryToolWithMessage(content); - - const result = await tool.execute("call-1", { sessionKey: "main" }); - const details = readHistoryDetails(result); - - expect(details.messages).toEqual([ - { - role: "user", - content: [{ type: "image", mimeType: "image/png", bytes: expectedBytes, omitted: true }], - }, - ]); - }); - it.each([0, 1.5])("rejects invalid limit value %s", async (limit) => { const tool = createHistoryToolWithMessage("hello"); diff --git a/src/agents/tools/sessions-history-tool.ts b/src/agents/tools/sessions-history-tool.ts index 0d371df00397..72ba054d2725 100644 --- a/src/agents/tools/sessions-history-tool.ts +++ b/src/agents/tools/sessions-history-tool.ts @@ -3,8 +3,6 @@ * * Reads bounded, redacted session transcript history after session visibility filtering. */ -import { estimateBase64DecodedBytes } from "@openclaw/media-core/base64"; -import { readStringValue } from "@openclaw/normalization-core/string-coerce"; import { Type } from "typebox"; import { getRuntimeConfig } from "../../config/config.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; @@ -123,29 +121,17 @@ function sanitizeHistoryContentBlock(block: unknown): { const entry = { ...(block as Record) }; let truncated = false; let redacted = false; - const type = typeof entry.type === "string" ? entry.type : ""; if (typeof entry.text === "string") { const res = truncateHistoryText(entry.text); entry.text = res.text; truncated ||= res.truncated; redacted ||= res.redacted; } - if (type === "thinking") { - if (typeof entry.thinking === "string") { - const res = truncateHistoryText(entry.thinking); - entry.thinking = res.text; - truncated ||= res.truncated; - redacted ||= res.redacted; - } - // The encrypted signature can be extremely large and is not useful for history recall. - if ("thinkingSignature" in entry) { - delete entry.thinkingSignature; - truncated = true; - } - if ("openclawReasoningReplay" in entry) { - delete entry.openclawReasoningReplay; - truncated = true; - } + if (entry.type === "thinking" && typeof entry.thinking === "string") { + const res = truncateHistoryText(entry.thinking); + entry.thinking = res.text; + truncated ||= res.truncated; + redacted ||= res.redacted; } if (typeof entry.partialJson === "string") { const res = truncateHistoryText(entry.partialJson); @@ -153,19 +139,6 @@ function sanitizeHistoryContentBlock(block: unknown): { truncated ||= res.truncated; redacted ||= res.redacted; } - if (type === "image") { - const data = readStringValue(entry.data); - const existingBytes = typeof entry.bytes === "number" ? entry.bytes : undefined; - const bytes = data === undefined ? existingBytes : estimateBase64DecodedBytes(data); - if ("data" in entry) { - delete entry.data; - truncated = true; - } - entry.omitted = true; - if (bytes !== undefined) { - entry.bytes = bytes; - } - } return { block: entry, truncated, redacted }; } @@ -180,10 +153,6 @@ function sanitizeHistoryMessage(message: unknown): { const entry = { ...(message as Record) }; let truncated = false; let redacted = false; - if ("providerReplay" in entry) { - delete entry.providerReplay; - redacted = true; - } // Tool result details often contain very large nested payloads. if ("details" in entry) { delete entry.details; diff --git a/src/gateway/chat-display-projection.sanitize.ts b/src/gateway/chat-display-projection.sanitize.ts index c7f8a531677a..8440bbc0a037 100644 --- a/src/gateway/chat-display-projection.sanitize.ts +++ b/src/gateway/chat-display-projection.sanitize.ts @@ -1,6 +1,7 @@ import { estimateBase64DecodedBytes } from "@openclaw/media-core/base64"; import { asFiniteNumber } from "@openclaw/normalization-core/number-coercion"; import { asOptionalRecord as readRecord } from "@openclaw/normalization-core/record-coerce"; +import { parseInboundMediaUri } from "../media/media-reference.js"; import { parseAssistantTextSignature, resolveAssistantMessagePhase, @@ -30,54 +31,109 @@ import { WORKSPACE_CONFLICT_TRANSCRIPT_TYPE, } from "./worker-environments/workspace-conflicts.js"; -const AUDIO_LOCAL_PATH_FIELDS = ["path", "file", "filePath", "localPath"] as const; +const MEDIA_PRIVATE_FIELDS = ["data", "blob", "path", "file", "filePath", "localPath"] as const; +const MEDIA_REFERENCE_FIELDS = ["url", "openUrl", "image_url", "audio_url", "video_url"] as const; +const MEDIA_FACT_PRIVATE_FIELDS = [ + "workspaceDir", + ...MEDIA_PRIVATE_FIELDS.filter((field) => field !== "path"), +] as const; -function isInlineOrLocalAudioReference(value: unknown): boolean { +function projectChatHistoryMediaReference(value: unknown): string | undefined { if (typeof value !== "string") { - return false; + return undefined; } const reference = value.trim(); - const isManagedRoute = /^\/(?:api\/chat\/media\/outgoing|media|__openclaw__)\//u.test(reference); - return ( - /^data:audio\//iu.test(reference) || - /^file:/iu.test(reference) || - /^~[\\/]/u.test(reference) || - (!isManagedRoute && - (reference.startsWith("/") || - /^[A-Za-z]:[\\/]/u.test(reference) || - reference.startsWith("\\\\"))) - ); + if (/^\/(?:api\/chat\/media\/outgoing|media|__openclaw__)\//u.test(reference)) { + return reference.split(/[?#]/u, 1)[0]; + } + try { + if (/^media:/iu.test(reference)) { + return parseInboundMediaUri(reference)?.normalizedSource; + } + const url = new URL(reference); + if (url.protocol !== "http:" && url.protocol !== "https:") { + return undefined; + } + url.username = url.password = url.search = url.hash = ""; + return url.toString(); + } catch { + return undefined; + } } -function omitAudioHistoryContent( - entry: Record, - referenceFields: readonly string[], -): boolean { - let removed = false; - if (Object.hasOwn(entry, "data")) { - const data = entry.data; - delete entry.data; - if (typeof data === "string") { - entry.bytes = estimateBase64DecodedBytes(data); - } - removed = true; +function projectChatHistoryMediaBlock(entry: Record, fact = false): boolean { + if (!fact && (typeof entry.type !== "string" || !/^(?:image|audio|video)$/u.test(entry.type))) { + return false; } - for (const field of AUDIO_LOCAL_PATH_FIELDS) { - if (Object.hasOwn(entry, field)) { - delete entry[field]; - removed = true; + const media = entry as typeof entry & { type: "image" | "audio" | "video" }; + const hasTopLevelPayload = typeof media.data === "string" || typeof media.blob === "string"; + const source = fact ? undefined : readRecord(media.source); + const projectedSource = source ? { ...source } : undefined; + const records: Record[] = [media, ...(projectedSource ? [projectedSource] : [])]; + if (projectedSource) { + media.source = projectedSource; + } + const privateFields = fact ? MEDIA_FACT_PRIVATE_FIELDS : MEDIA_PRIVATE_FIELDS; + const referenceFields = fact ? (["path", "url"] as const) : MEDIA_REFERENCE_FIELDS; + const sourceIsReference = + !source && + (!fact || + typeof media.source !== "string" || + /^(?:[a-z][a-z0-9+.-]*:|~?[\\/])|[\\/]/iu.test(media.source)); + let encodedPayload: string | undefined; + for (const record of records) { + let omitted = false; + const payload = typeof record.data === "string" ? record.data : record.blob; + if (encodedPayload === undefined && typeof payload === "string") { + encodedPayload = payload; + } + for (const field of privateFields) { + if (!Object.hasOwn(record, field)) { + continue; + } + delete record[field]; + omitted = true; + } + const recordReferences = + record === media && sourceIsReference ? [...referenceFields, "source"] : referenceFields; + for (const field of recordReferences) { + if (!Object.hasOwn(record, field)) { + continue; + } + const projected = projectChatHistoryMediaReference(record[field]); + record[field] = projected; + if (projected === undefined) { + delete record[field]; + omitted = true; + } + } + if (!fact && omitted) { + // Preserve shipped image/audio omission ownership; new video blocks mark both levels. + if (record === media || media.type !== "image") { + record.omitted = true; + } + if (record === media || media.type !== "audio") { + media.omitted = true; + } } } - for (const field of referenceFields) { - if (isInlineOrLocalAudioReference(entry[field])) { - delete entry[field]; - removed = true; - } + if (!fact && encodedPayload !== undefined) { + (media.type === "audio" && !hasTopLevelPayload && projectedSource + ? projectedSource + : media + ).bytes = estimateBase64DecodedBytes(encodedPayload); } - if (removed) { - entry.omitted = true; - } - return removed; + return true; +} + +function projectChatHistoryMediaFacts(value: unknown): unknown[] | undefined { + return Array.isArray(value) + ? value.map((fact) => { + const projected = { ...readRecord(fact) }; + projectChatHistoryMediaBlock(projected, true); + return projected; + }) + : undefined; } export function sanitizeChatHistoryContentBlock( @@ -146,37 +202,8 @@ export function sanitizeChatHistoryContentBlock( delete entry.openclawReasoningReplay; changed = true; } - const type = typeof entry.type === "string" ? entry.type : ""; - if (type === "image") { - let imageData = typeof entry.data === "string" ? entry.data : undefined; - const source = readRecord(entry.source); - if (source?.type === "base64" && typeof source.data === "string") { - imageData ??= source.data; - const projectedSource = { ...source }; - delete projectedSource.data; - entry.source = projectedSource; - } - if (imageData !== undefined) { - delete entry.data; - entry.omitted = true; - entry.bytes = estimateBase64DecodedBytes(imageData); - changed = true; - } - } - if (type === "audio") { - // Audio transcripts can retain model-input bytes and host-local references. - // Strip them at the shared display boundary while preserving safe metadata. - const blockChanged = omitAudioHistoryContent(entry, ["url", "openUrl", "audio_url"]); - changed ||= blockChanged; - const source = readRecord(entry.source); - if (source) { - const projectedSource = { ...source }; - if (omitAudioHistoryContent(projectedSource, ["url"])) { - entry.source = projectedSource; - changed = true; - } - } - } + const mediaChanged = projectChatHistoryMediaBlock(entry); + changed ||= mediaChanged; return { block: changed ? entry : block, changed }; } @@ -360,11 +387,17 @@ export function sanitizeChatHistoryMessage( changed = true; } const openClawMeta = readRecord(entry["__openclaw"]); - if (openClawMeta && "upstreamUserText" in openClawMeta) { + if (openClawMeta && ("upstreamUserText" in openClawMeta || "media" in openClawMeta)) { // Codex retains the decorated upstream prompt for transcript reconstruction. // It is not display data and can otherwise evict the visible row from history. const projectedMeta = { ...openClawMeta }; delete projectedMeta.upstreamUserText; + if ("media" in projectedMeta) { + projectedMeta.media = projectChatHistoryMediaFacts(projectedMeta.media); + if (projectedMeta.media === undefined) { + delete projectedMeta.media; + } + } if (Object.keys(projectedMeta).length > 0) { entry["__openclaw"] = projectedMeta; } else { diff --git a/src/gateway/chat-display-projection.test.ts b/src/gateway/chat-display-projection.test.ts index fbd1fe0ab93b..8ce83c718e93 100644 --- a/src/gateway/chat-display-projection.test.ts +++ b/src/gateway/chat-display-projection.test.ts @@ -21,6 +21,148 @@ function projectHistoryTransports(message: Record) { } describe("oversized multimodal chat history", () => { + it("projects one mixed-media message through every history boundary", async () => { + const inlineImage = Buffer.from("inline image").toString("base64"); + const inlineAudio = Buffer.from("inline audio").toString("base64"); + const inlineVideo = Buffer.from("inline video").toString("base64"); + const rawMessage = { + role: "user", + content: [ + { type: "text", text: "keep mixed media metadata" }, + { + type: "image", + mimeType: "image/png", + data: inlineImage, + path: "/tmp/private-image.png", + url: "https://image-user@media.example/image.png?signature=image-secret#image-fragment", + source: { + type: "base64", + data: inlineImage, + blob: inlineImage, + url: "media://inbound/image-claim", + }, + }, + { + type: "audio", + mimeType: "audio/wav", + blob: inlineAudio, + filePath: String.raw`C:\private-audio.wav`, + audio_url: "media://inbound/audio-claim", + source: { + type: "url", + data: inlineAudio, + url: "https://audio-user@media.example/audio.wav?token=audio-secret#audio-fragment", + }, + }, + { + type: "video", + mimeType: "video/mp4", + data: inlineVideo, + localPath: String.raw`\\server\share\private-video.mp4`, + video_url: + "https://video-user@media.example/video.mp4?X-Amz-Signature=video-secret#video-fragment", + source: { + type: "url", + blob: inlineVideo, + url: "media://inbound/video-claim", + }, + }, + ], + }; + const expected = projectChatDisplayMessages([rawMessage]); + const snapshot = buildSessionHistorySnapshot({ rawMessages: [rawMessage] }).history.messages; + const sseState = SessionHistorySseState.fromRawSnapshot({ + target: { sessionId: "mixed-media", sessionKey: "agent:main:mixed-media" }, + rawMessages: [], + }); + const incremental = sseState.appendInlineMessage({ message: rawMessage })?.message; + const projections = [ + ["projectChatDisplayMessages", expected], + ["session-history snapshot", snapshot], + ["incremental SSE state", incremental ? [incremental] : []], + ] as const; + for (const [boundary, messages] of projections) { + expect(messages, boundary).toHaveLength(1); + expect(messages[0], boundary).toMatchObject({ + role: "user", + content: expected[0]?.content, + }); + const serialized = JSON.stringify(messages); + for (const secret of [ + inlineImage, + inlineAudio, + inlineVideo, + "private-image", + "private-audio", + "private-video", + "image-user", + "audio-user", + "video-user", + "image-secret", + "audio-secret", + "video-secret", + "image-fragment", + "audio-fragment", + "video-fragment", + ]) { + expect(serialized, `${boundary}: ${secret}`).not.toContain(secret); + } + expect(serialized, boundary).toContain("media://inbound/image-claim"); + expect(serialized, boundary).toContain("media://inbound/audio-claim"); + expect(serialized, boundary).toContain("media://inbound/video-claim"); + expect(serialized, boundary).toContain("https://media.example/image.png"); + expect(serialized, boundary).toContain("https://media.example/audio.wav"); + expect(serialized, boundary).toContain("https://media.example/video.mp4"); + } + }); + + it("projects media even when another block field is sanitized first", () => { + const payload = Buffer.from("short-circuit video payload"); + const encoded = payload.toString("base64"); + const message = { + role: "user", + content: [ + { + type: "video", + mimeType: "video/mp4", + data: encoded, + blob: encoded, + path: "/private/short-circuit-video.mp4", + url: "https://media-user@media.example/video.mp4?signature=private-signature#private-fragment", + openclawReasoningReplay: { private: true }, + }, + ], + }; + + const messages = sanitizeChatHistoryMessages([message]); + + expect(messages).toEqual([ + { + role: "user", + content: [ + { + type: "video", + mimeType: "video/mp4", + url: "https://media.example/video.mp4", + omitted: true, + bytes: payload.length, + }, + ], + }, + ]); + const serialized = JSON.stringify(messages); + for (const privateValue of [ + encoded, + "/private/short-circuit-video.mp4", + "media-user", + "private-signature", + "private-fragment", + "openclawReasoningReplay", + ]) { + expect(serialized).not.toContain(privateValue); + } + }); + it.each([ { name: "native image data", diff --git a/src/gateway/server-methods/server-methods.test.ts b/src/gateway/server-methods/server-methods.test.ts index fc6cf53ec530..9d84f4d17616 100644 --- a/src/gateway/server-methods/server-methods.test.ts +++ b/src/gateway/server-methods/server-methods.test.ts @@ -1996,13 +1996,19 @@ describe("projectRecentChatDisplayMessages", () => { it.each([ { name: "facts-only", - message: { __openclaw: { media: [{ path: "/tmp/openclaw/fact.png" }] } }, - expectedPath: "/tmp/openclaw/fact.png", + message: { + __openclaw: { media: [{ path: "/tmp/openclaw/fact.png", contentType: "image/png" }] }, + }, + expectedPath: undefined, }, { name: "sparse", - message: { __openclaw: { media: [{}, { path: "/tmp/openclaw/sparse.png" }] } }, - expectedPath: "/tmp/openclaw/sparse.png", + message: { + __openclaw: { + media: [{}, { path: "/tmp/openclaw/sparse.png", contentType: "image/png" }], + }, + }, + expectedPath: undefined, expectedIndex: 1, }, { @@ -2012,8 +2018,12 @@ describe("projectRecentChatDisplayMessages", () => { }, { name: "media-only", - message: { __openclaw: { media: [{ path: "/tmp/openclaw/media-only.png" }] } }, - expectedPath: "/tmp/openclaw/media-only.png", + message: { + __openclaw: { + media: [{ path: "/tmp/openclaw/media-only.png", contentType: "image/png" }], + }, + }, + expectedPath: undefined, }, ])("keeps $name media-only users through canonical display projection", (testCase) => { const result = projectRecentChatDisplayMessages([ diff --git a/src/gateway/server.chat.gateway-server-chat-b.test.ts b/src/gateway/server.chat.gateway-server-chat-b.test.ts index 9594cb14531c..0dad65b9b956 100644 --- a/src/gateway/server.chat.gateway-server-chat-b.test.ts +++ b/src/gateway/server.chat.gateway-server-chat-b.test.ts @@ -7,6 +7,7 @@ import { afterAll, afterEach, beforeAll, describe, expect, test, vi } from "vite import { createDeferred } from "../../test/helpers/promise.js"; import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; import type { ModelCatalogEntry } from "../agents/model-catalog.types.js"; +import { createSessionsHistoryTool } from "../agents/tools/sessions-history-tool.js"; import type { GetReplyOptions } from "../auto-reply/get-reply-options.types.js"; import { HEARTBEAT_PROMPT } from "../auto-reply/heartbeat.js"; import type { InternalGetReplyOptions } from "../auto-reply/reply/get-reply.types.js"; @@ -35,6 +36,7 @@ import { isSessionWorkAdmissionActive, runExclusiveSessionLifecycleMutation, } from "../sessions/session-lifecycle-admission.js"; +import { buildPersistedUserTurnMessage } from "../sessions/user-turn-transcript.js"; import { openOpenClawAgentDatabase } from "../state/openclaw-agent-db.js"; import { captureEnv, setTestEnvValue } from "../test-utils/env.js"; import { withOpenClawTestState } from "../test-utils/openclaw-test-state.js"; @@ -5065,6 +5067,171 @@ describe("gateway server chat", () => { }); }); + test("projects persisted media facts through Gateway history and sessions_history", async () => { + await withGatewayChatHarness(async ({ ws, createSessionDir }) => { + await prepareMainHistoryHarness({ ws, createSessionDir }); + const invalidClaims = [ + "media://inbound/nested/file.png", + "media://inbound/nested%2Ffile.png", + "media://inbound/nested%5Cfile.png", + "media://inbound/file%00.png", + "media://inbound/", + "media://inbound/.", + "media://inbound/..", + ["media://user", "password@inbound/claim.png"].join(":"), + "media://inbound/claim.png?signature=private-secret", + "media://inbound/claim.png#private-fragment", + ]; + const persisted = buildPersistedUserTurnMessage({ + text: "inspect mixed attachments", + timestamp: Date.now(), + media: [ + { + kind: "image", + path: "/private/media/local-image.png", + workspaceDir: "/private/workspace", + contentType: "image/png", + fileName: "local-image.png", + sizeBytes: 42, + width: 640, + height: 480, + messageId: "local-source-id", + }, + { + kind: "audio", + url: "https://media-user@media.example/audio.wav?signature=private-signature#audio-fragment", + contentType: "audio/wav", + fileName: "remote-audio.wav", + durationMs: 1234, + }, + { + kind: "video", + path: "media://inbound/video-claim", + contentType: "video/mp4", + fileName: "managed-video.mp4", + durationMs: 5678, + }, + { + kind: "document", + url: "not a media reference", + contentType: "application/pdf", + fileName: "metadata-only.pdf", + }, + ...invalidClaims.map((claim, index) => ({ + kind: "image" as const, + path: claim, + contentType: "image/png", + fileName: `invalid-claim-${index}.png`, + })), + ], + }) as unknown as Record; + const metadata = persisted["__openclaw"] as Record; + const facts = metadata.media as Array>; + Object.assign(expectDefined(facts[0], "local media fact"), { + data: "private-inline-data", + blob: "private-inline-blob", + filePath: "/private/media/alternate-image.png", + source: "telegram-attachment-1", + }); + metadata.upstreamUserText = "private upstream prompt"; + metadata.keepMe = { durable: true }; + await writeMainSessionTranscript([{ id: "persisted-media", message: persisted }]); + + const historyMessages = await fetchHistoryMessages(ws); + const tool = createSessionsHistoryTool({ + config: {}, + callGateway: async >(request: { + method: string; + params?: unknown; + }) => { + const response = await rpcReq>( + ws, + request.method, + request.params, + ); + expect(response.ok).toBe(true); + return expectDefined(response.payload, `${request.method} payload`); + }, + }); + const toolResult = await tool.execute("persisted-media", { sessionKey: "main" }); + const sessionsHistory = (toolResult.details as { messages: unknown[] }).messages; + + for (const [boundary, messages] of [ + ["chat.history", historyMessages], + ["sessions_history", sessionsHistory], + ] as const) { + expect(messages, boundary).toHaveLength(1); + expect(messages[0], boundary).toMatchObject({ + role: "user", + content: "inspect mixed attachments", + __openclaw: { + keepMe: { durable: true }, + media: [ + { + kind: "image", + contentType: "image/png", + fileName: "local-image.png", + sizeBytes: 42, + width: 640, + height: 480, + messageId: "local-source-id", + source: "telegram-attachment-1", + }, + { + kind: "audio", + url: "https://media.example/audio.wav", + contentType: "audio/wav", + fileName: "remote-audio.wav", + durationMs: 1234, + }, + { + kind: "video", + path: "media://inbound/video-claim", + contentType: "video/mp4", + fileName: "managed-video.mp4", + durationMs: 5678, + }, + { + kind: "document", + contentType: "application/pdf", + fileName: "metadata-only.pdf", + }, + ...invalidClaims.map((_, index) => ({ + kind: "image", + contentType: "image/png", + fileName: `invalid-claim-${index}.png`, + })), + ], + }, + }); + const projectedMedia = ( + (messages[0] as { __openclaw?: { media?: Array> } })["__openclaw"] + ?.media ?? [] + ).map((fact) => fact.path ?? fact.url ?? null); + expect(projectedMedia, boundary).toEqual([ + null, + "https://media.example/audio.wav", + "media://inbound/video-claim", + ...Array.from({ length: invalidClaims.length + 1 }, () => null), + ]); + const serialized = JSON.stringify(messages); + for (const privateValue of [ + "/private/media", + "/private/workspace", + "private-inline-data", + "private-inline-blob", + "media-user", + "private-signature", + "audio-fragment", + "private upstream prompt", + "not a media reference", + ]) { + expect(serialized, `${boundary}: ${privateValue}`).not.toContain(privateValue); + } + } + }); + }); + test("chat.history keeps recent messages within the production byte budget", async () => { await withGatewayChatHarness(async ({ ws, createSessionDir }) => { await prepareMainHistoryHarness({ ws, createSessionDir }); diff --git a/src/media/media-reference.test.ts b/src/media/media-reference.test.ts index afbe457af7f5..1be36e569e43 100644 --- a/src/media/media-reference.test.ts +++ b/src/media/media-reference.test.ts @@ -214,6 +214,13 @@ describe("media reference helpers", () => { MediaReferenceError, ); expect(() => parseInboundMediaUri("media://inbound/%00.png")).toThrow(MediaReferenceError); + for (const claim of [ + ["media://user", "password@inbound/claim.png"].join(":"), + "media://inbound/claim.png?signature=private-secret", + "media://inbound/claim.png#private-fragment", + ]) { + expect(() => parseInboundMediaUri(claim), claim).toThrow(MediaReferenceError); + } }); it("rejects symlinked inbound media files", async () => { diff --git a/src/media/media-reference.ts b/src/media/media-reference.ts index 91b0a06be1cb..3b9997591b63 100644 --- a/src/media/media-reference.ts +++ b/src/media/media-reference.ts @@ -136,6 +136,9 @@ export function parseInboundMediaUri(source: string): InboundMediaUri | null { `Unsupported media URI location: ${parsed.hostname || "(missing)"}`, ); } + if (parsed.username || parsed.password || parsed.search || parsed.hash) { + throw new MediaReferenceError("invalid-path", `Invalid media URI: ${normalizedSource}`); + } let id: string; try { @@ -146,7 +149,8 @@ export function parseInboundMediaUri(source: string): InboundMediaUri | null { }); } - if (!id || id.includes("/") || id.includes("\\") || id.includes("\0")) { + const invalidId = !id || id === "." || id === ".."; + if (invalidId || id.includes("/") || id.includes("\\") || id.includes("\0")) { throw new MediaReferenceError("invalid-path", `Invalid media URI: ${normalizedSource}`); } diff --git a/ui/src/lib/chat/message-extract.ts b/ui/src/lib/chat/message-extract.ts index 4e7d62c2c1ba..7f873b0671c1 100644 --- a/ui/src/lib/chat/message-extract.ts +++ b/ui/src/lib/chat/message-extract.ts @@ -1,10 +1,7 @@ // Control UI chat module implements message extract behavior. import { stripInternalRuntimeContext } from "../../../../src/agents/internal-runtime-context.js"; import { stripInboundMetadata } from "../../../../src/auto-reply/reply/strip-inbound-meta.js"; -import { - isMeaningfulMediaFact, - readPersistedMediaFacts, -} from "../../../../src/media/media-facts.js"; +import { readPersistedMediaFacts } from "../../../../src/media/media-facts.js"; import { stripEnvelope } from "../../../../src/shared/chat-envelope.js"; import { extractAssistantVisibleText as extractSharedAssistantVisibleText } from "../../../../src/shared/chat-message-content.js"; import { normalizeLowercaseStringOrEmpty } from "../string-coerce.ts"; @@ -125,12 +122,6 @@ export function extractRawText(message: unknown): string | null { return null; } -function hasTranscriptMediaFacts(message: unknown): boolean { - return message != null && typeof message === "object" - ? (readPersistedMediaFacts(message) ?? []).some(isMeaningfulMediaFact) - : false; -} - export function readTranscriptMediaEntries(message: unknown): Array<{ path: string; mediaType: string | undefined; @@ -196,7 +187,7 @@ export function isEmptyUserTextOnlyMessage(message: unknown): boolean { if (normalizeLowercaseStringOrEmpty(entry.role) !== "user") { return false; } - if (hasTranscriptMediaFacts(entry)) { + if (readTranscriptMediaEntries(entry).length > 0) { return false; } if (!isTextOnlyContent(entry.content ?? entry.text)) { diff --git a/ui/src/pages/chat/chat-history.media.test.ts b/ui/src/pages/chat/chat-history.media.test.ts index 96bd2abbc234..bea07c995814 100644 --- a/ui/src/pages/chat/chat-history.media.test.ts +++ b/ui/src/pages/chat/chat-history.media.test.ts @@ -3,6 +3,7 @@ import { isEmptyUserTextOnlyMessage, readTranscriptMediaEntries, } from "../../lib/chat/message-extract.ts"; +import { buildChatItems } from "./chat-thread-build.ts"; import { extractTranscriptAttachments } from "./components/chat-message-media.ts"; const MANAGED_UUID = "43007e90-2ade-43f2-a781-42b843e9eca3"; @@ -19,7 +20,6 @@ describe("chat history canonical media filtering", () => { it.each([ ["facts-only", [{ path: "/media/fact.png", contentType: "image/png" }]], ["sparse", [{}, { path: "/media/sparse.png", contentType: "image/png" }]], - ["type-only", [{ contentType: "image/png" }]], ["media-only", [{ url: "media://inbound/media-only.png", kind: "image" }]], ])("keeps an empty %s user row", (_name, media) => { expect( @@ -31,8 +31,32 @@ describe("chat history canonical media filtering", () => { ).toBe(false); }); - it("drops a truly empty user row", () => { - expect(isEmptyUserTextOnlyMessage({ role: "user", content: "" })).toBe(true); + it.each([ + ["truly empty", { role: "user", content: "" }], + ["metadata-only media", userMessageWithMedia([{ contentType: "image/png" }])], + ])("drops a %s user row", (_name, message) => { + expect(isEmptyUserTextOnlyMessage(message)).toBe(true); + }); + + it("renders a safe media-only user turn without rendering metadata-only local media", () => { + const safeRef = "media://inbound/safe-history-image.png"; + const items = buildChatItems({ + paneId: "media-history", + sessionKey: "main", + messages: [ + userMessageWithMedia([{ path: safeRef, contentType: "image/png" }]), + userMessageWithMedia([{ contentType: "image/png", fileName: "metadata-only-local.png" }]), + ], + toolMessages: [], + streamSegments: [], + stream: null, + streamStartedAt: null, + showToolCalls: true, + }); + const serialized = JSON.stringify(items); + + expect(serialized).toContain(safeRef); + expect(serialized).not.toContain("metadata-only-local.png"); }); }); diff --git a/ui/src/pages/chat/chat-thread-items.ts b/ui/src/pages/chat/chat-thread-items.ts index ea1c9cd7b2bc..d6dc5f3dad25 100644 --- a/ui/src/pages/chat/chat-thread-items.ts +++ b/ui/src/pages/chat/chat-thread-items.ts @@ -8,7 +8,7 @@ import type { NormalizedMessage, ToolCard, } from "../../lib/chat/chat-types.ts"; -import { extractTextCached } from "../../lib/chat/message-extract.ts"; +import { extractTextCached, readTranscriptMediaEntries } from "../../lib/chat/message-extract.ts"; import { normalizeMessage, stripMessageDisplayMetadataText, @@ -561,8 +561,9 @@ export function hasRenderableNormalizedMessage(message: unknown): boolean { return false; } const role = normalizeRoleForGrouping(normalized.role); - const hasVisibleSenderLabel = role === "assistant" && Boolean(normalized.senderLabel?.trim()); - return normalized.content.length > 0 || Boolean(normalized.replyTarget) || hasVisibleSenderLabel; + const label = role === "assistant" && normalized.senderLabel?.trim(); + const media = role === "user" && readTranscriptMediaEntries(message).length; + return Boolean(normalized.content.length || normalized.replyTarget || label || media); } export function sanitizeStreamText(text: string): string {