mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
fix(gateway): unify media privacy in chat history (#121490)
* fix(gateway): unify media privacy in chat history Centralize image, audio, video, and persisted media-fact privacy at the shared Gateway history projection. Remove duplicate sessions_history redaction, validate managed media claims canonically, and keep safe media-only user turns renderable. * test(gateway): type history RPC integration * test(agents): align history fixture with gateway projection
This commit is contained in:
Peter Steinberger
committed by
GitHub
GitHub
parent
a4eabd5744
commit
0a6a1d9419
@@ -737,7 +737,7 @@ describe("sessions tools", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("sessions_history caps oversized payloads and strips heavy fields", async () => {
|
||||
it("sessions_history caps oversized payloads and strips tool-owned heavy fields", async () => {
|
||||
const oversized = Array.from({ length: 80 }, (_, idx) => ({
|
||||
role: "assistant",
|
||||
content: [
|
||||
@@ -748,14 +748,6 @@ describe("sessions tools", () => {
|
||||
{
|
||||
type: "thinking",
|
||||
thinking: "y".repeat(7000),
|
||||
thinkingSignature: "sig".repeat(4000),
|
||||
openclawReasoningReplay: {
|
||||
v: 1,
|
||||
source: "openai-responses",
|
||||
provider: "openai",
|
||||
api: "openai-chatgpt-responses",
|
||||
model: "gpt-5.5",
|
||||
},
|
||||
},
|
||||
],
|
||||
details: {
|
||||
@@ -765,10 +757,6 @@ describe("sessions tools", () => {
|
||||
input: 1,
|
||||
output: 1,
|
||||
},
|
||||
providerReplay: {
|
||||
type: "openai-responses-compaction",
|
||||
data: "opaque-sessions-history-compaction",
|
||||
},
|
||||
}));
|
||||
callGatewayMock.mockImplementation(async (opts: unknown) => {
|
||||
const request = opts as { method?: string };
|
||||
@@ -795,7 +783,7 @@ describe("sessions tools", () => {
|
||||
expect(details.truncated).toBe(true);
|
||||
expect(details.droppedMessages).toBe(true);
|
||||
expect(details.contentTruncated).toBe(true);
|
||||
expect(details.contentRedacted).toBe(true);
|
||||
expect(details.contentRedacted).toBe(false);
|
||||
expect(typeof details.bytes).toBe("number");
|
||||
expect((details.bytes ?? 0) <= 80 * 1024).toBe(true);
|
||||
expect(details.messages && details.messages.length > 0).toBe(true);
|
||||
@@ -804,26 +792,20 @@ describe("sessions tools", () => {
|
||||
| {
|
||||
details?: unknown;
|
||||
usage?: unknown;
|
||||
providerReplay?: unknown;
|
||||
content?: Array<{
|
||||
type?: string;
|
||||
text?: string;
|
||||
thinking?: string;
|
||||
thinkingSignature?: string;
|
||||
openclawReasoningReplay?: unknown;
|
||||
}>;
|
||||
}
|
||||
| undefined;
|
||||
expect(first?.details).toBeUndefined();
|
||||
expect(first?.usage).toBeUndefined();
|
||||
expect(first?.providerReplay).toBeUndefined();
|
||||
expect(JSON.stringify(details.messages)).not.toContain("opaque-sessions-history-compaction");
|
||||
const textBlock = first?.content?.find((block) => block.type === "text");
|
||||
expect(typeof textBlock?.text).toBe("string");
|
||||
expect((textBlock?.text ?? "").length <= 4015).toBe(true);
|
||||
const thinkingBlock = first?.content?.find((block) => block.type === "thinking");
|
||||
expect(thinkingBlock?.thinkingSignature).toBeUndefined();
|
||||
expect(thinkingBlock?.openclawReasoningReplay).toBeUndefined();
|
||||
expect((thinkingBlock?.thinking ?? "").length <= 4015).toBe(true);
|
||||
});
|
||||
|
||||
it("sessions_history enforces a hard byte cap even when a single message is huge", async () => {
|
||||
|
||||
@@ -175,42 +175,6 @@ describe("sessions_history redaction", () => {
|
||||
expect((result.details as { contentRedacted?: unknown }).contentRedacted).toBe(true);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
name: "reports decoded bytes for raw image data",
|
||||
content: [
|
||||
{
|
||||
type: "image",
|
||||
data: Buffer.from([0, 1, 2, 3, 4]).toString("base64"),
|
||||
mimeType: "image/png",
|
||||
},
|
||||
],
|
||||
expectedBytes: 5,
|
||||
},
|
||||
{
|
||||
name: "replaces stale bytes for empty image data",
|
||||
content: [{ type: "image", data: "", mimeType: "image/png", bytes: 999 }],
|
||||
expectedBytes: 0,
|
||||
},
|
||||
{
|
||||
name: "preserves existing bytes when image data is already omitted",
|
||||
content: [{ type: "image", mimeType: "image/png", bytes: 37, omitted: true }],
|
||||
expectedBytes: 37,
|
||||
},
|
||||
])("$name", async ({ content, expectedBytes }) => {
|
||||
const tool = createHistoryToolWithMessage(content);
|
||||
|
||||
const result = await tool.execute("call-1", { sessionKey: "main" });
|
||||
const details = readHistoryDetails(result);
|
||||
|
||||
expect(details.messages).toEqual([
|
||||
{
|
||||
role: "user",
|
||||
content: [{ type: "image", mimeType: "image/png", bytes: expectedBytes, omitted: true }],
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it.each([0, 1.5])("rejects invalid limit value %s", async (limit) => {
|
||||
const tool = createHistoryToolWithMessage("hello");
|
||||
|
||||
|
||||
@@ -3,8 +3,6 @@
|
||||
*
|
||||
* Reads bounded, redacted session transcript history after session visibility filtering.
|
||||
*/
|
||||
import { estimateBase64DecodedBytes } from "@openclaw/media-core/base64";
|
||||
import { readStringValue } from "@openclaw/normalization-core/string-coerce";
|
||||
import { Type } from "typebox";
|
||||
import { getRuntimeConfig } from "../../config/config.js";
|
||||
import type { OpenClawConfig } from "../../config/types.openclaw.js";
|
||||
@@ -123,29 +121,17 @@ function sanitizeHistoryContentBlock(block: unknown): {
|
||||
const entry = { ...(block as Record<string, unknown>) };
|
||||
let truncated = false;
|
||||
let redacted = false;
|
||||
const type = typeof entry.type === "string" ? entry.type : "";
|
||||
if (typeof entry.text === "string") {
|
||||
const res = truncateHistoryText(entry.text);
|
||||
entry.text = res.text;
|
||||
truncated ||= res.truncated;
|
||||
redacted ||= res.redacted;
|
||||
}
|
||||
if (type === "thinking") {
|
||||
if (typeof entry.thinking === "string") {
|
||||
const res = truncateHistoryText(entry.thinking);
|
||||
entry.thinking = res.text;
|
||||
truncated ||= res.truncated;
|
||||
redacted ||= res.redacted;
|
||||
}
|
||||
// The encrypted signature can be extremely large and is not useful for history recall.
|
||||
if ("thinkingSignature" in entry) {
|
||||
delete entry.thinkingSignature;
|
||||
truncated = true;
|
||||
}
|
||||
if ("openclawReasoningReplay" in entry) {
|
||||
delete entry.openclawReasoningReplay;
|
||||
truncated = true;
|
||||
}
|
||||
if (entry.type === "thinking" && typeof entry.thinking === "string") {
|
||||
const res = truncateHistoryText(entry.thinking);
|
||||
entry.thinking = res.text;
|
||||
truncated ||= res.truncated;
|
||||
redacted ||= res.redacted;
|
||||
}
|
||||
if (typeof entry.partialJson === "string") {
|
||||
const res = truncateHistoryText(entry.partialJson);
|
||||
@@ -153,19 +139,6 @@ function sanitizeHistoryContentBlock(block: unknown): {
|
||||
truncated ||= res.truncated;
|
||||
redacted ||= res.redacted;
|
||||
}
|
||||
if (type === "image") {
|
||||
const data = readStringValue(entry.data);
|
||||
const existingBytes = typeof entry.bytes === "number" ? entry.bytes : undefined;
|
||||
const bytes = data === undefined ? existingBytes : estimateBase64DecodedBytes(data);
|
||||
if ("data" in entry) {
|
||||
delete entry.data;
|
||||
truncated = true;
|
||||
}
|
||||
entry.omitted = true;
|
||||
if (bytes !== undefined) {
|
||||
entry.bytes = bytes;
|
||||
}
|
||||
}
|
||||
return { block: entry, truncated, redacted };
|
||||
}
|
||||
|
||||
@@ -180,10 +153,6 @@ function sanitizeHistoryMessage(message: unknown): {
|
||||
const entry = { ...(message as Record<string, unknown>) };
|
||||
let truncated = false;
|
||||
let redacted = false;
|
||||
if ("providerReplay" in entry) {
|
||||
delete entry.providerReplay;
|
||||
redacted = true;
|
||||
}
|
||||
// Tool result details often contain very large nested payloads.
|
||||
if ("details" in entry) {
|
||||
delete entry.details;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { estimateBase64DecodedBytes } from "@openclaw/media-core/base64";
|
||||
import { asFiniteNumber } from "@openclaw/normalization-core/number-coercion";
|
||||
import { asOptionalRecord as readRecord } from "@openclaw/normalization-core/record-coerce";
|
||||
import { parseInboundMediaUri } from "../media/media-reference.js";
|
||||
import {
|
||||
parseAssistantTextSignature,
|
||||
resolveAssistantMessagePhase,
|
||||
@@ -30,54 +31,109 @@ import {
|
||||
WORKSPACE_CONFLICT_TRANSCRIPT_TYPE,
|
||||
} from "./worker-environments/workspace-conflicts.js";
|
||||
|
||||
const AUDIO_LOCAL_PATH_FIELDS = ["path", "file", "filePath", "localPath"] as const;
|
||||
const MEDIA_PRIVATE_FIELDS = ["data", "blob", "path", "file", "filePath", "localPath"] as const;
|
||||
const MEDIA_REFERENCE_FIELDS = ["url", "openUrl", "image_url", "audio_url", "video_url"] as const;
|
||||
const MEDIA_FACT_PRIVATE_FIELDS = [
|
||||
"workspaceDir",
|
||||
...MEDIA_PRIVATE_FIELDS.filter((field) => field !== "path"),
|
||||
] as const;
|
||||
|
||||
function isInlineOrLocalAudioReference(value: unknown): boolean {
|
||||
function projectChatHistoryMediaReference(value: unknown): string | undefined {
|
||||
if (typeof value !== "string") {
|
||||
return false;
|
||||
return undefined;
|
||||
}
|
||||
const reference = value.trim();
|
||||
const isManagedRoute = /^\/(?:api\/chat\/media\/outgoing|media|__openclaw__)\//u.test(reference);
|
||||
return (
|
||||
/^data:audio\//iu.test(reference) ||
|
||||
/^file:/iu.test(reference) ||
|
||||
/^~[\\/]/u.test(reference) ||
|
||||
(!isManagedRoute &&
|
||||
(reference.startsWith("/") ||
|
||||
/^[A-Za-z]:[\\/]/u.test(reference) ||
|
||||
reference.startsWith("\\\\")))
|
||||
);
|
||||
if (/^\/(?:api\/chat\/media\/outgoing|media|__openclaw__)\//u.test(reference)) {
|
||||
return reference.split(/[?#]/u, 1)[0];
|
||||
}
|
||||
try {
|
||||
if (/^media:/iu.test(reference)) {
|
||||
return parseInboundMediaUri(reference)?.normalizedSource;
|
||||
}
|
||||
const url = new URL(reference);
|
||||
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
||||
return undefined;
|
||||
}
|
||||
url.username = url.password = url.search = url.hash = "";
|
||||
return url.toString();
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function omitAudioHistoryContent(
|
||||
entry: Record<string, unknown>,
|
||||
referenceFields: readonly string[],
|
||||
): boolean {
|
||||
let removed = false;
|
||||
if (Object.hasOwn(entry, "data")) {
|
||||
const data = entry.data;
|
||||
delete entry.data;
|
||||
if (typeof data === "string") {
|
||||
entry.bytes = estimateBase64DecodedBytes(data);
|
||||
}
|
||||
removed = true;
|
||||
function projectChatHistoryMediaBlock(entry: Record<string, unknown>, fact = false): boolean {
|
||||
if (!fact && (typeof entry.type !== "string" || !/^(?:image|audio|video)$/u.test(entry.type))) {
|
||||
return false;
|
||||
}
|
||||
for (const field of AUDIO_LOCAL_PATH_FIELDS) {
|
||||
if (Object.hasOwn(entry, field)) {
|
||||
delete entry[field];
|
||||
removed = true;
|
||||
const media = entry as typeof entry & { type: "image" | "audio" | "video" };
|
||||
const hasTopLevelPayload = typeof media.data === "string" || typeof media.blob === "string";
|
||||
const source = fact ? undefined : readRecord(media.source);
|
||||
const projectedSource = source ? { ...source } : undefined;
|
||||
const records: Record<string, unknown>[] = [media, ...(projectedSource ? [projectedSource] : [])];
|
||||
if (projectedSource) {
|
||||
media.source = projectedSource;
|
||||
}
|
||||
const privateFields = fact ? MEDIA_FACT_PRIVATE_FIELDS : MEDIA_PRIVATE_FIELDS;
|
||||
const referenceFields = fact ? (["path", "url"] as const) : MEDIA_REFERENCE_FIELDS;
|
||||
const sourceIsReference =
|
||||
!source &&
|
||||
(!fact ||
|
||||
typeof media.source !== "string" ||
|
||||
/^(?:[a-z][a-z0-9+.-]*:|~?[\\/])|[\\/]/iu.test(media.source));
|
||||
let encodedPayload: string | undefined;
|
||||
for (const record of records) {
|
||||
let omitted = false;
|
||||
const payload = typeof record.data === "string" ? record.data : record.blob;
|
||||
if (encodedPayload === undefined && typeof payload === "string") {
|
||||
encodedPayload = payload;
|
||||
}
|
||||
for (const field of privateFields) {
|
||||
if (!Object.hasOwn(record, field)) {
|
||||
continue;
|
||||
}
|
||||
delete record[field];
|
||||
omitted = true;
|
||||
}
|
||||
const recordReferences =
|
||||
record === media && sourceIsReference ? [...referenceFields, "source"] : referenceFields;
|
||||
for (const field of recordReferences) {
|
||||
if (!Object.hasOwn(record, field)) {
|
||||
continue;
|
||||
}
|
||||
const projected = projectChatHistoryMediaReference(record[field]);
|
||||
record[field] = projected;
|
||||
if (projected === undefined) {
|
||||
delete record[field];
|
||||
omitted = true;
|
||||
}
|
||||
}
|
||||
if (!fact && omitted) {
|
||||
// Preserve shipped image/audio omission ownership; new video blocks mark both levels.
|
||||
if (record === media || media.type !== "image") {
|
||||
record.omitted = true;
|
||||
}
|
||||
if (record === media || media.type !== "audio") {
|
||||
media.omitted = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const field of referenceFields) {
|
||||
if (isInlineOrLocalAudioReference(entry[field])) {
|
||||
delete entry[field];
|
||||
removed = true;
|
||||
}
|
||||
if (!fact && encodedPayload !== undefined) {
|
||||
(media.type === "audio" && !hasTopLevelPayload && projectedSource
|
||||
? projectedSource
|
||||
: media
|
||||
).bytes = estimateBase64DecodedBytes(encodedPayload);
|
||||
}
|
||||
if (removed) {
|
||||
entry.omitted = true;
|
||||
}
|
||||
return removed;
|
||||
return true;
|
||||
}
|
||||
|
||||
function projectChatHistoryMediaFacts(value: unknown): unknown[] | undefined {
|
||||
return Array.isArray(value)
|
||||
? value.map((fact) => {
|
||||
const projected = { ...readRecord(fact) };
|
||||
projectChatHistoryMediaBlock(projected, true);
|
||||
return projected;
|
||||
})
|
||||
: undefined;
|
||||
}
|
||||
|
||||
export function sanitizeChatHistoryContentBlock(
|
||||
@@ -146,37 +202,8 @@ export function sanitizeChatHistoryContentBlock(
|
||||
delete entry.openclawReasoningReplay;
|
||||
changed = true;
|
||||
}
|
||||
const type = typeof entry.type === "string" ? entry.type : "";
|
||||
if (type === "image") {
|
||||
let imageData = typeof entry.data === "string" ? entry.data : undefined;
|
||||
const source = readRecord(entry.source);
|
||||
if (source?.type === "base64" && typeof source.data === "string") {
|
||||
imageData ??= source.data;
|
||||
const projectedSource = { ...source };
|
||||
delete projectedSource.data;
|
||||
entry.source = projectedSource;
|
||||
}
|
||||
if (imageData !== undefined) {
|
||||
delete entry.data;
|
||||
entry.omitted = true;
|
||||
entry.bytes = estimateBase64DecodedBytes(imageData);
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
if (type === "audio") {
|
||||
// Audio transcripts can retain model-input bytes and host-local references.
|
||||
// Strip them at the shared display boundary while preserving safe metadata.
|
||||
const blockChanged = omitAudioHistoryContent(entry, ["url", "openUrl", "audio_url"]);
|
||||
changed ||= blockChanged;
|
||||
const source = readRecord(entry.source);
|
||||
if (source) {
|
||||
const projectedSource = { ...source };
|
||||
if (omitAudioHistoryContent(projectedSource, ["url"])) {
|
||||
entry.source = projectedSource;
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
const mediaChanged = projectChatHistoryMediaBlock(entry);
|
||||
changed ||= mediaChanged;
|
||||
return { block: changed ? entry : block, changed };
|
||||
}
|
||||
|
||||
@@ -360,11 +387,17 @@ export function sanitizeChatHistoryMessage(
|
||||
changed = true;
|
||||
}
|
||||
const openClawMeta = readRecord(entry["__openclaw"]);
|
||||
if (openClawMeta && "upstreamUserText" in openClawMeta) {
|
||||
if (openClawMeta && ("upstreamUserText" in openClawMeta || "media" in openClawMeta)) {
|
||||
// Codex retains the decorated upstream prompt for transcript reconstruction.
|
||||
// It is not display data and can otherwise evict the visible row from history.
|
||||
const projectedMeta = { ...openClawMeta };
|
||||
delete projectedMeta.upstreamUserText;
|
||||
if ("media" in projectedMeta) {
|
||||
projectedMeta.media = projectChatHistoryMediaFacts(projectedMeta.media);
|
||||
if (projectedMeta.media === undefined) {
|
||||
delete projectedMeta.media;
|
||||
}
|
||||
}
|
||||
if (Object.keys(projectedMeta).length > 0) {
|
||||
entry["__openclaw"] = projectedMeta;
|
||||
} else {
|
||||
|
||||
@@ -21,6 +21,148 @@ function projectHistoryTransports(message: Record<string, unknown>) {
|
||||
}
|
||||
|
||||
describe("oversized multimodal chat history", () => {
|
||||
it("projects one mixed-media message through every history boundary", async () => {
|
||||
const inlineImage = Buffer.from("inline image").toString("base64");
|
||||
const inlineAudio = Buffer.from("inline audio").toString("base64");
|
||||
const inlineVideo = Buffer.from("inline video").toString("base64");
|
||||
const rawMessage = {
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: "keep mixed media metadata" },
|
||||
{
|
||||
type: "image",
|
||||
mimeType: "image/png",
|
||||
data: inlineImage,
|
||||
path: "/tmp/private-image.png",
|
||||
url: "https://image-user@media.example/image.png?signature=image-secret#image-fragment",
|
||||
source: {
|
||||
type: "base64",
|
||||
data: inlineImage,
|
||||
blob: inlineImage,
|
||||
url: "media://inbound/image-claim",
|
||||
},
|
||||
},
|
||||
{
|
||||
type: "audio",
|
||||
mimeType: "audio/wav",
|
||||
blob: inlineAudio,
|
||||
filePath: String.raw`C:\private-audio.wav`,
|
||||
audio_url: "media://inbound/audio-claim",
|
||||
source: {
|
||||
type: "url",
|
||||
data: inlineAudio,
|
||||
url: "https://audio-user@media.example/audio.wav?token=audio-secret#audio-fragment",
|
||||
},
|
||||
},
|
||||
{
|
||||
type: "video",
|
||||
mimeType: "video/mp4",
|
||||
data: inlineVideo,
|
||||
localPath: String.raw`\\server\share\private-video.mp4`,
|
||||
video_url:
|
||||
"https://video-user@media.example/video.mp4?X-Amz-Signature=video-secret#video-fragment",
|
||||
source: {
|
||||
type: "url",
|
||||
blob: inlineVideo,
|
||||
url: "media://inbound/video-claim",
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
const expected = projectChatDisplayMessages([rawMessage]);
|
||||
const snapshot = buildSessionHistorySnapshot({ rawMessages: [rawMessage] }).history.messages;
|
||||
const sseState = SessionHistorySseState.fromRawSnapshot({
|
||||
target: { sessionId: "mixed-media", sessionKey: "agent:main:mixed-media" },
|
||||
rawMessages: [],
|
||||
});
|
||||
const incremental = sseState.appendInlineMessage({ message: rawMessage })?.message;
|
||||
const projections = [
|
||||
["projectChatDisplayMessages", expected],
|
||||
["session-history snapshot", snapshot],
|
||||
["incremental SSE state", incremental ? [incremental] : []],
|
||||
] as const;
|
||||
for (const [boundary, messages] of projections) {
|
||||
expect(messages, boundary).toHaveLength(1);
|
||||
expect(messages[0], boundary).toMatchObject({
|
||||
role: "user",
|
||||
content: expected[0]?.content,
|
||||
});
|
||||
const serialized = JSON.stringify(messages);
|
||||
for (const secret of [
|
||||
inlineImage,
|
||||
inlineAudio,
|
||||
inlineVideo,
|
||||
"private-image",
|
||||
"private-audio",
|
||||
"private-video",
|
||||
"image-user",
|
||||
"audio-user",
|
||||
"video-user",
|
||||
"image-secret",
|
||||
"audio-secret",
|
||||
"video-secret",
|
||||
"image-fragment",
|
||||
"audio-fragment",
|
||||
"video-fragment",
|
||||
]) {
|
||||
expect(serialized, `${boundary}: ${secret}`).not.toContain(secret);
|
||||
}
|
||||
expect(serialized, boundary).toContain("media://inbound/image-claim");
|
||||
expect(serialized, boundary).toContain("media://inbound/audio-claim");
|
||||
expect(serialized, boundary).toContain("media://inbound/video-claim");
|
||||
expect(serialized, boundary).toContain("https://media.example/image.png");
|
||||
expect(serialized, boundary).toContain("https://media.example/audio.wav");
|
||||
expect(serialized, boundary).toContain("https://media.example/video.mp4");
|
||||
}
|
||||
});
|
||||
|
||||
it("projects media even when another block field is sanitized first", () => {
|
||||
const payload = Buffer.from("short-circuit video payload");
|
||||
const encoded = payload.toString("base64");
|
||||
const message = {
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
type: "video",
|
||||
mimeType: "video/mp4",
|
||||
data: encoded,
|
||||
blob: encoded,
|
||||
path: "/private/short-circuit-video.mp4",
|
||||
url: "https://media-user@media.example/video.mp4?signature=private-signature#private-fragment",
|
||||
openclawReasoningReplay: { private: true },
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const messages = sanitizeChatHistoryMessages([message]);
|
||||
|
||||
expect(messages).toEqual([
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
type: "video",
|
||||
mimeType: "video/mp4",
|
||||
url: "https://media.example/video.mp4",
|
||||
omitted: true,
|
||||
bytes: payload.length,
|
||||
},
|
||||
],
|
||||
},
|
||||
]);
|
||||
const serialized = JSON.stringify(messages);
|
||||
for (const privateValue of [
|
||||
encoded,
|
||||
"/private/short-circuit-video.mp4",
|
||||
"media-user",
|
||||
"private-signature",
|
||||
"private-fragment",
|
||||
"openclawReasoningReplay",
|
||||
]) {
|
||||
expect(serialized).not.toContain(privateValue);
|
||||
}
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
name: "native image data",
|
||||
|
||||
@@ -1996,13 +1996,19 @@ describe("projectRecentChatDisplayMessages", () => {
|
||||
it.each([
|
||||
{
|
||||
name: "facts-only",
|
||||
message: { __openclaw: { media: [{ path: "/tmp/openclaw/fact.png" }] } },
|
||||
expectedPath: "/tmp/openclaw/fact.png",
|
||||
message: {
|
||||
__openclaw: { media: [{ path: "/tmp/openclaw/fact.png", contentType: "image/png" }] },
|
||||
},
|
||||
expectedPath: undefined,
|
||||
},
|
||||
{
|
||||
name: "sparse",
|
||||
message: { __openclaw: { media: [{}, { path: "/tmp/openclaw/sparse.png" }] } },
|
||||
expectedPath: "/tmp/openclaw/sparse.png",
|
||||
message: {
|
||||
__openclaw: {
|
||||
media: [{}, { path: "/tmp/openclaw/sparse.png", contentType: "image/png" }],
|
||||
},
|
||||
},
|
||||
expectedPath: undefined,
|
||||
expectedIndex: 1,
|
||||
},
|
||||
{
|
||||
@@ -2012,8 +2018,12 @@ describe("projectRecentChatDisplayMessages", () => {
|
||||
},
|
||||
{
|
||||
name: "media-only",
|
||||
message: { __openclaw: { media: [{ path: "/tmp/openclaw/media-only.png" }] } },
|
||||
expectedPath: "/tmp/openclaw/media-only.png",
|
||||
message: {
|
||||
__openclaw: {
|
||||
media: [{ path: "/tmp/openclaw/media-only.png", contentType: "image/png" }],
|
||||
},
|
||||
},
|
||||
expectedPath: undefined,
|
||||
},
|
||||
])("keeps $name media-only users through canonical display projection", (testCase) => {
|
||||
const result = projectRecentChatDisplayMessages([
|
||||
|
||||
@@ -7,6 +7,7 @@ import { afterAll, afterEach, beforeAll, describe, expect, test, vi } from "vite
|
||||
import { createDeferred } from "../../test/helpers/promise.js";
|
||||
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
|
||||
import type { ModelCatalogEntry } from "../agents/model-catalog.types.js";
|
||||
import { createSessionsHistoryTool } from "../agents/tools/sessions-history-tool.js";
|
||||
import type { GetReplyOptions } from "../auto-reply/get-reply-options.types.js";
|
||||
import { HEARTBEAT_PROMPT } from "../auto-reply/heartbeat.js";
|
||||
import type { InternalGetReplyOptions } from "../auto-reply/reply/get-reply.types.js";
|
||||
@@ -35,6 +36,7 @@ import {
|
||||
isSessionWorkAdmissionActive,
|
||||
runExclusiveSessionLifecycleMutation,
|
||||
} from "../sessions/session-lifecycle-admission.js";
|
||||
import { buildPersistedUserTurnMessage } from "../sessions/user-turn-transcript.js";
|
||||
import { openOpenClawAgentDatabase } from "../state/openclaw-agent-db.js";
|
||||
import { captureEnv, setTestEnvValue } from "../test-utils/env.js";
|
||||
import { withOpenClawTestState } from "../test-utils/openclaw-test-state.js";
|
||||
@@ -5065,6 +5067,171 @@ describe("gateway server chat", () => {
|
||||
});
|
||||
});
|
||||
|
||||
test("projects persisted media facts through Gateway history and sessions_history", async () => {
|
||||
await withGatewayChatHarness(async ({ ws, createSessionDir }) => {
|
||||
await prepareMainHistoryHarness({ ws, createSessionDir });
|
||||
const invalidClaims = [
|
||||
"media://inbound/nested/file.png",
|
||||
"media://inbound/nested%2Ffile.png",
|
||||
"media://inbound/nested%5Cfile.png",
|
||||
"media://inbound/file%00.png",
|
||||
"media://inbound/",
|
||||
"media://inbound/.",
|
||||
"media://inbound/..",
|
||||
["media://user", "password@inbound/claim.png"].join(":"),
|
||||
"media://inbound/claim.png?signature=private-secret",
|
||||
"media://inbound/claim.png#private-fragment",
|
||||
];
|
||||
const persisted = buildPersistedUserTurnMessage({
|
||||
text: "inspect mixed attachments",
|
||||
timestamp: Date.now(),
|
||||
media: [
|
||||
{
|
||||
kind: "image",
|
||||
path: "/private/media/local-image.png",
|
||||
workspaceDir: "/private/workspace",
|
||||
contentType: "image/png",
|
||||
fileName: "local-image.png",
|
||||
sizeBytes: 42,
|
||||
width: 640,
|
||||
height: 480,
|
||||
messageId: "local-source-id",
|
||||
},
|
||||
{
|
||||
kind: "audio",
|
||||
url: "https://media-user@media.example/audio.wav?signature=private-signature#audio-fragment",
|
||||
contentType: "audio/wav",
|
||||
fileName: "remote-audio.wav",
|
||||
durationMs: 1234,
|
||||
},
|
||||
{
|
||||
kind: "video",
|
||||
path: "media://inbound/video-claim",
|
||||
contentType: "video/mp4",
|
||||
fileName: "managed-video.mp4",
|
||||
durationMs: 5678,
|
||||
},
|
||||
{
|
||||
kind: "document",
|
||||
url: "not a media reference",
|
||||
contentType: "application/pdf",
|
||||
fileName: "metadata-only.pdf",
|
||||
},
|
||||
...invalidClaims.map((claim, index) => ({
|
||||
kind: "image" as const,
|
||||
path: claim,
|
||||
contentType: "image/png",
|
||||
fileName: `invalid-claim-${index}.png`,
|
||||
})),
|
||||
],
|
||||
}) as unknown as Record<string, unknown>;
|
||||
const metadata = persisted["__openclaw"] as Record<string, unknown>;
|
||||
const facts = metadata.media as Array<Record<string, unknown>>;
|
||||
Object.assign(expectDefined(facts[0], "local media fact"), {
|
||||
data: "private-inline-data",
|
||||
blob: "private-inline-blob",
|
||||
filePath: "/private/media/alternate-image.png",
|
||||
source: "telegram-attachment-1",
|
||||
});
|
||||
metadata.upstreamUserText = "private upstream prompt";
|
||||
metadata.keepMe = { durable: true };
|
||||
await writeMainSessionTranscript([{ id: "persisted-media", message: persisted }]);
|
||||
|
||||
const historyMessages = await fetchHistoryMessages(ws);
|
||||
const tool = createSessionsHistoryTool({
|
||||
config: {},
|
||||
callGateway: async <T = Record<string, unknown>>(request: {
|
||||
method: string;
|
||||
params?: unknown;
|
||||
}) => {
|
||||
const response = await rpcReq<T & Record<string, unknown>>(
|
||||
ws,
|
||||
request.method,
|
||||
request.params,
|
||||
);
|
||||
expect(response.ok).toBe(true);
|
||||
return expectDefined(response.payload, `${request.method} payload`);
|
||||
},
|
||||
});
|
||||
const toolResult = await tool.execute("persisted-media", { sessionKey: "main" });
|
||||
const sessionsHistory = (toolResult.details as { messages: unknown[] }).messages;
|
||||
|
||||
for (const [boundary, messages] of [
|
||||
["chat.history", historyMessages],
|
||||
["sessions_history", sessionsHistory],
|
||||
] as const) {
|
||||
expect(messages, boundary).toHaveLength(1);
|
||||
expect(messages[0], boundary).toMatchObject({
|
||||
role: "user",
|
||||
content: "inspect mixed attachments",
|
||||
__openclaw: {
|
||||
keepMe: { durable: true },
|
||||
media: [
|
||||
{
|
||||
kind: "image",
|
||||
contentType: "image/png",
|
||||
fileName: "local-image.png",
|
||||
sizeBytes: 42,
|
||||
width: 640,
|
||||
height: 480,
|
||||
messageId: "local-source-id",
|
||||
source: "telegram-attachment-1",
|
||||
},
|
||||
{
|
||||
kind: "audio",
|
||||
url: "https://media.example/audio.wav",
|
||||
contentType: "audio/wav",
|
||||
fileName: "remote-audio.wav",
|
||||
durationMs: 1234,
|
||||
},
|
||||
{
|
||||
kind: "video",
|
||||
path: "media://inbound/video-claim",
|
||||
contentType: "video/mp4",
|
||||
fileName: "managed-video.mp4",
|
||||
durationMs: 5678,
|
||||
},
|
||||
{
|
||||
kind: "document",
|
||||
contentType: "application/pdf",
|
||||
fileName: "metadata-only.pdf",
|
||||
},
|
||||
...invalidClaims.map((_, index) => ({
|
||||
kind: "image",
|
||||
contentType: "image/png",
|
||||
fileName: `invalid-claim-${index}.png`,
|
||||
})),
|
||||
],
|
||||
},
|
||||
});
|
||||
const projectedMedia = (
|
||||
(messages[0] as { __openclaw?: { media?: Array<Record<string, unknown>> } })["__openclaw"]
|
||||
?.media ?? []
|
||||
).map((fact) => fact.path ?? fact.url ?? null);
|
||||
expect(projectedMedia, boundary).toEqual([
|
||||
null,
|
||||
"https://media.example/audio.wav",
|
||||
"media://inbound/video-claim",
|
||||
...Array.from({ length: invalidClaims.length + 1 }, () => null),
|
||||
]);
|
||||
const serialized = JSON.stringify(messages);
|
||||
for (const privateValue of [
|
||||
"/private/media",
|
||||
"/private/workspace",
|
||||
"private-inline-data",
|
||||
"private-inline-blob",
|
||||
"media-user",
|
||||
"private-signature",
|
||||
"audio-fragment",
|
||||
"private upstream prompt",
|
||||
"not a media reference",
|
||||
]) {
|
||||
expect(serialized, `${boundary}: ${privateValue}`).not.toContain(privateValue);
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("chat.history keeps recent messages within the production byte budget", async () => {
|
||||
await withGatewayChatHarness(async ({ ws, createSessionDir }) => {
|
||||
await prepareMainHistoryHarness({ ws, createSessionDir });
|
||||
|
||||
@@ -214,6 +214,13 @@ describe("media reference helpers", () => {
|
||||
MediaReferenceError,
|
||||
);
|
||||
expect(() => parseInboundMediaUri("media://inbound/%00.png")).toThrow(MediaReferenceError);
|
||||
for (const claim of [
|
||||
["media://user", "password@inbound/claim.png"].join(":"),
|
||||
"media://inbound/claim.png?signature=private-secret",
|
||||
"media://inbound/claim.png#private-fragment",
|
||||
]) {
|
||||
expect(() => parseInboundMediaUri(claim), claim).toThrow(MediaReferenceError);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects symlinked inbound media files", async () => {
|
||||
|
||||
@@ -136,6 +136,9 @@ export function parseInboundMediaUri(source: string): InboundMediaUri | null {
|
||||
`Unsupported media URI location: ${parsed.hostname || "(missing)"}`,
|
||||
);
|
||||
}
|
||||
if (parsed.username || parsed.password || parsed.search || parsed.hash) {
|
||||
throw new MediaReferenceError("invalid-path", `Invalid media URI: ${normalizedSource}`);
|
||||
}
|
||||
|
||||
let id: string;
|
||||
try {
|
||||
@@ -146,7 +149,8 @@ export function parseInboundMediaUri(source: string): InboundMediaUri | null {
|
||||
});
|
||||
}
|
||||
|
||||
if (!id || id.includes("/") || id.includes("\\") || id.includes("\0")) {
|
||||
const invalidId = !id || id === "." || id === "..";
|
||||
if (invalidId || id.includes("/") || id.includes("\\") || id.includes("\0")) {
|
||||
throw new MediaReferenceError("invalid-path", `Invalid media URI: ${normalizedSource}`);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,10 +1,7 @@
|
||||
// Control UI chat module implements message extract behavior.
|
||||
import { stripInternalRuntimeContext } from "../../../../src/agents/internal-runtime-context.js";
|
||||
import { stripInboundMetadata } from "../../../../src/auto-reply/reply/strip-inbound-meta.js";
|
||||
import {
|
||||
isMeaningfulMediaFact,
|
||||
readPersistedMediaFacts,
|
||||
} from "../../../../src/media/media-facts.js";
|
||||
import { readPersistedMediaFacts } from "../../../../src/media/media-facts.js";
|
||||
import { stripEnvelope } from "../../../../src/shared/chat-envelope.js";
|
||||
import { extractAssistantVisibleText as extractSharedAssistantVisibleText } from "../../../../src/shared/chat-message-content.js";
|
||||
import { normalizeLowercaseStringOrEmpty } from "../string-coerce.ts";
|
||||
@@ -125,12 +122,6 @@ export function extractRawText(message: unknown): string | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
function hasTranscriptMediaFacts(message: unknown): boolean {
|
||||
return message != null && typeof message === "object"
|
||||
? (readPersistedMediaFacts(message) ?? []).some(isMeaningfulMediaFact)
|
||||
: false;
|
||||
}
|
||||
|
||||
export function readTranscriptMediaEntries(message: unknown): Array<{
|
||||
path: string;
|
||||
mediaType: string | undefined;
|
||||
@@ -196,7 +187,7 @@ export function isEmptyUserTextOnlyMessage(message: unknown): boolean {
|
||||
if (normalizeLowercaseStringOrEmpty(entry.role) !== "user") {
|
||||
return false;
|
||||
}
|
||||
if (hasTranscriptMediaFacts(entry)) {
|
||||
if (readTranscriptMediaEntries(entry).length > 0) {
|
||||
return false;
|
||||
}
|
||||
if (!isTextOnlyContent(entry.content ?? entry.text)) {
|
||||
|
||||
@@ -3,6 +3,7 @@ import {
|
||||
isEmptyUserTextOnlyMessage,
|
||||
readTranscriptMediaEntries,
|
||||
} from "../../lib/chat/message-extract.ts";
|
||||
import { buildChatItems } from "./chat-thread-build.ts";
|
||||
import { extractTranscriptAttachments } from "./components/chat-message-media.ts";
|
||||
|
||||
const MANAGED_UUID = "43007e90-2ade-43f2-a781-42b843e9eca3";
|
||||
@@ -19,7 +20,6 @@ describe("chat history canonical media filtering", () => {
|
||||
it.each([
|
||||
["facts-only", [{ path: "/media/fact.png", contentType: "image/png" }]],
|
||||
["sparse", [{}, { path: "/media/sparse.png", contentType: "image/png" }]],
|
||||
["type-only", [{ contentType: "image/png" }]],
|
||||
["media-only", [{ url: "media://inbound/media-only.png", kind: "image" }]],
|
||||
])("keeps an empty %s user row", (_name, media) => {
|
||||
expect(
|
||||
@@ -31,8 +31,32 @@ describe("chat history canonical media filtering", () => {
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("drops a truly empty user row", () => {
|
||||
expect(isEmptyUserTextOnlyMessage({ role: "user", content: "" })).toBe(true);
|
||||
it.each([
|
||||
["truly empty", { role: "user", content: "" }],
|
||||
["metadata-only media", userMessageWithMedia([{ contentType: "image/png" }])],
|
||||
])("drops a %s user row", (_name, message) => {
|
||||
expect(isEmptyUserTextOnlyMessage(message)).toBe(true);
|
||||
});
|
||||
|
||||
it("renders a safe media-only user turn without rendering metadata-only local media", () => {
|
||||
const safeRef = "media://inbound/safe-history-image.png";
|
||||
const items = buildChatItems({
|
||||
paneId: "media-history",
|
||||
sessionKey: "main",
|
||||
messages: [
|
||||
userMessageWithMedia([{ path: safeRef, contentType: "image/png" }]),
|
||||
userMessageWithMedia([{ contentType: "image/png", fileName: "metadata-only-local.png" }]),
|
||||
],
|
||||
toolMessages: [],
|
||||
streamSegments: [],
|
||||
stream: null,
|
||||
streamStartedAt: null,
|
||||
showToolCalls: true,
|
||||
});
|
||||
const serialized = JSON.stringify(items);
|
||||
|
||||
expect(serialized).toContain(safeRef);
|
||||
expect(serialized).not.toContain("metadata-only-local.png");
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ import type {
|
||||
NormalizedMessage,
|
||||
ToolCard,
|
||||
} from "../../lib/chat/chat-types.ts";
|
||||
import { extractTextCached } from "../../lib/chat/message-extract.ts";
|
||||
import { extractTextCached, readTranscriptMediaEntries } from "../../lib/chat/message-extract.ts";
|
||||
import {
|
||||
normalizeMessage,
|
||||
stripMessageDisplayMetadataText,
|
||||
@@ -561,8 +561,9 @@ export function hasRenderableNormalizedMessage(message: unknown): boolean {
|
||||
return false;
|
||||
}
|
||||
const role = normalizeRoleForGrouping(normalized.role);
|
||||
const hasVisibleSenderLabel = role === "assistant" && Boolean(normalized.senderLabel?.trim());
|
||||
return normalized.content.length > 0 || Boolean(normalized.replyTarget) || hasVisibleSenderLabel;
|
||||
const label = role === "assistant" && normalized.senderLabel?.trim();
|
||||
const media = role === "user" && readTranscriptMediaEntries(message).length;
|
||||
return Boolean(normalized.content.length || normalized.replyTarget || label || media);
|
||||
}
|
||||
|
||||
export function sanitizeStreamText(text: string): string {
|
||||
|
||||
Reference in New Issue
Block a user