refactor(media): introduce neutral MediaFact owner with identity-preserving adapters (#112197)

* refactor(media): introduce neutral MediaFact owner with identity-preserving adapters

Media-facts program PR 1 (audit-frozen plan): canonical MediaFact
ownership and normalization/projection helpers in a neutral core media
module; legacy channel/plugin/agent payload builders become adapters
that project exactly their declared input fields (structural richer
inputs cannot leak url/kind into historically path/contentType-only
outputs); persisted transcript input keeps accepting arbitrary stored
kind strings, narrowing only during runtime-fact normalization.

* fix(media): restore SDK and dead-code gates
This commit is contained in:
Peter Steinberger
2026-07-21 00:50:41 -07:00
committed by GitHub
parent 9e0c5f94b5
commit 5abcad51c8
10 changed files with 255 additions and 181 deletions
@@ -27,7 +27,7 @@ ad60ccc4fe9084d47f0477e02d9296bacad32f26d7456e2a84be8d25a53a25c2 module/boolean
a6bc51efd3ce23e65432c9a6790670e2a0acb96ae7263fedbaed52ea353228be module/channel-core
fbf353eb38ae68d8ded3f2a60b432c7bb2c245d2ec7e7c9f53c6da19a0db0938 module/channel-entry-contract
982f29a18e07228e3da82cae67d06ff38249592a29c2fd28f01f0d2016ff80d9 module/channel-feedback
7de503d739b127b23af2c7f13eb6064f69dfb9f29d1ba7bd8d98c524a3b7257d module/channel-inbound
d1df68d6c5773da6113f8b237dc75316ec1fb878c807cb00d86070aadadccabf module/channel-inbound
2940039d5ebdb16c1d745914390db01c0f5f1e9cff33f95ea36c9e77d97c2bae module/channel-inbound-debounce
91425ad75ddb58a3796f7bc88570cb2c81b3db0dd6521a7ce0e55f9148880a47 module/channel-ingress-runtime
c97dd36cdf8f83c2893c33e9430a93cd131a03d855725783ca5b545de0cf84f8 module/channel-lifecycle
+6 -7
View File
@@ -1,3 +1,5 @@
import type { MediaFact } from "../../media/media-facts.js";
/** Normalized history message used when building reply context. */
export type HistoryEntry = {
sender: string;
@@ -8,10 +10,7 @@ export type HistoryEntry = {
};
/** Media metadata attached to a normalized history message. */
export type HistoryMediaEntry = {
path?: string;
url?: string;
contentType?: string;
kind?: import("@openclaw/media-core/constants").MediaKind;
messageId?: string;
};
export type HistoryMediaEntry = Pick<
MediaFact,
"contentType" | "kind" | "messageId" | "path" | "url"
>;
+94 -33
View File
@@ -1,12 +1,16 @@
// Inbound event media tests cover channel media attachment normalization.
import { describe, expect, it } from "vitest";
import { normalizeAttachments } from "../../media-understanding/attachments.normalize.js";
import { normalizeMediaFacts, projectMediaFacts } from "../../media/media-facts.js";
import { buildAgentMediaPayload } from "../../plugin-sdk/agent-media-payload.js";
import { buildMediaPayload } from "../plugins/media-payload.js";
import {
buildChannelInboundMediaPayload,
formatMediaPlaceholderText,
formatInboundMediaUnavailableText,
toHistoryMediaEntries,
toInboundMediaFacts,
type ChannelInboundMediaInput,
} from "./media.js";
describe("channel inbound media facts", () => {
@@ -61,27 +65,20 @@ describe("channel inbound media facts", () => {
});
it("normalizes provider media into inbound media facts", () => {
expect(
toInboundMediaFacts(
[
{
path: " /tmp/image.png ",
contentType: " image/png ",
messageId: " ",
},
{
url: "https://example.test/audio.mp3",
contentType: "audio/mpeg",
kind: "audio",
},
],
{
kind: "image",
messageId: "msg-1",
transcribed: (_media, index) => index === 1,
},
),
).toEqual([
const input = [
{ path: " /tmp/image.png ", contentType: " image/png ", messageId: " " },
{
url: "https://example.test/audio.mp3",
contentType: "audio/mpeg",
kind: "audio" as const,
},
];
const defaults = {
kind: "image" as const,
messageId: "msg-1",
transcribed: (_media: ChannelInboundMediaInput, index: number): boolean => index === 1,
};
const expected = [
{
path: "/tmp/image.png",
url: undefined,
@@ -98,21 +95,35 @@ describe("channel inbound media facts", () => {
transcribed: true,
messageId: "msg-1",
},
];
expect(normalizeMediaFacts(input, defaults)).toEqual(expected);
expect(toInboundMediaFacts(input, defaults)).toEqual(expected);
expect(
normalizeMediaFacts([{ path: " image.png ", workspaceDir: " /tmp/workspace " }]),
).toEqual([
{
path: "image.png",
url: undefined,
contentType: undefined,
kind: undefined,
transcribed: false,
messageId: undefined,
workspaceDir: "/tmp/workspace",
},
]);
});
it("builds legacy media payload fields from inbound media facts", () => {
expect(
buildChannelInboundMediaPayload([
{ path: "/tmp/image.png", contentType: "image/png", kind: "image" },
{
url: "https://example.test/audio.mp3",
contentType: "audio/mpeg",
kind: "audio",
transcribed: true,
},
]),
).toEqual({
const media = [
{ path: "/tmp/image.png", contentType: "image/png", kind: "image" as const },
{
url: "https://example.test/audio.mp3",
contentType: "audio/mpeg",
kind: "audio" as const,
transcribed: true,
},
];
const expected = {
MediaPath: "/tmp/image.png",
MediaUrl: "/tmp/image.png",
MediaType: "image/png",
@@ -120,7 +131,9 @@ describe("channel inbound media facts", () => {
MediaUrls: ["/tmp/image.png", "https://example.test/audio.mp3"],
MediaTypes: ["image/png", "audio/mpeg"],
MediaTranscribedIndexes: [1],
});
};
expect(projectMediaFacts(media)).toEqual(expected);
expect(buildChannelInboundMediaPayload(media)).toEqual(expected);
});
it("keeps legacy media arrays index-aligned for mixed path and URL media", () => {
@@ -138,6 +151,54 @@ describe("channel inbound media facts", () => {
]);
});
it("keeps compact and cardinality-preserving adapter projections byte-identical", () => {
const media = [{ path: "/tmp/image.png", contentType: "image/png" }, { path: "/tmp/file.bin" }];
const compact = {
MediaPath: "/tmp/image.png",
MediaUrl: "/tmp/image.png",
MediaType: "image/png",
MediaPaths: ["/tmp/image.png", "/tmp/file.bin"],
MediaUrls: ["/tmp/image.png", "/tmp/file.bin"],
MediaTypes: ["image/png"],
};
expect(projectMediaFacts(media, "compact")).toEqual(compact);
expect(buildAgentMediaPayload(media)).toEqual(compact);
expect(buildMediaPayload(media)).toEqual(compact);
const aligned = { ...compact, MediaTypes: ["image/png", ""] };
expect(projectMediaFacts(media, "aligned")).toEqual(aligned);
expect(buildMediaPayload(media, { preserveMediaTypeCardinality: true })).toEqual(aligned);
});
it("keeps richer fact fields out of legacy outbound payloads", () => {
const richerMedia = [
{
path: "/tmp/voice-note.ogg",
url: "https://example.test/voice-note.ogg",
kind: "audio" as const,
},
];
const compact = {
MediaPath: "/tmp/voice-note.ogg",
MediaUrl: "/tmp/voice-note.ogg",
MediaType: undefined,
MediaPaths: ["/tmp/voice-note.ogg"],
MediaUrls: ["/tmp/voice-note.ogg"],
MediaTypes: undefined,
};
expect(projectMediaFacts(richerMedia, "compact")).toEqual(compact);
expect(buildAgentMediaPayload(richerMedia)).toEqual(compact);
expect(buildMediaPayload(richerMedia)).toEqual(compact);
expect(projectMediaFacts(richerMedia, "aligned")).toEqual({
...compact,
MediaTypes: [""],
});
expect(buildMediaPayload(richerMedia, { preserveMediaTypeCardinality: true })).toEqual({
...compact,
MediaTypes: [""],
});
});
it("maps inbound media facts into history media entries", () => {
expect(
toHistoryMediaEntries([{ path: "/tmp/image.png", contentType: "image/png" }], {
+14 -69
View File
@@ -1,16 +1,14 @@
import { kindFromMime, mimeTypeFromFilePath } from "@openclaw/media-core/mime";
/**
* Channel inbound media normalization.
*
* Converts plugin attachment metadata into aligned prompt/context media payload fields.
*/
import { normalizeOptionalString as normalizeString } from "@openclaw/normalization-core/string-coerce";
/** Channel inbound media normalization and compatibility projection. */
import type { HistoryMediaEntry } from "../../auto-reply/reply/history.types.js";
import {
normalizeMediaFacts,
projectMediaFacts,
type MediaFactLegacyProjection,
} from "../../media/media-facts.js";
import type { InboundMediaFacts } from "../turn/types.js";
/**
* Attachment metadata accepted from channel plugins before core normalization.
*/
/** Attachment metadata accepted from channel plugins before core normalization. */
export type ChannelInboundMediaInput = {
path?: string | null;
url?: string | null;
@@ -66,17 +64,9 @@ export function formatMediaPlaceholderText(media: readonly MediaPlaceholderTextF
: `${tag} (${media.length} ${PLURAL_MEDIA_PLACEHOLDER_LABELS[kind]})`;
}
/**
* Environment payload fields consumed by prompt/context builders for inbound media attachments.
*/
/** Legacy environment fields consumed by prompt/context builders. */
export type ChannelInboundMediaPayload = {
MediaPath?: string;
MediaUrl?: string;
MediaType?: string;
MediaPaths?: string[];
MediaUrls?: string[];
MediaTypes?: string[];
MediaTranscribedIndexes?: number[];
[Key in keyof MediaFactLegacyProjection]: MediaFactLegacyProjection[Key];
};
/** Appends an unavailable-media notice to real caption text, or returns the notice alone. */
@@ -92,26 +82,7 @@ export function formatInboundMediaUnavailableText(params: {
return `${body}\n\n${notice}`;
}
function alignedStrings(values: Array<string | undefined>): string[] | undefined {
if (!values.some(Boolean)) {
return undefined;
}
// Preserve indexes across parallel Media* arrays so transcribed indexes and
// media metadata continue to refer to the same attachment.
return values.map((value) => value ?? "");
}
function normalizeKind(value: InboundMediaFacts["kind"] | null | undefined) {
return value ?? undefined;
}
function mediaType(media: InboundMediaFacts): string | undefined {
return media.contentType ?? media.kind;
}
/**
* Normalizes plugin-provided attachment facts into the channel turn media shape.
*/
/** Normalizes plugin-provided attachments into ordered runtime facts. */
export function toInboundMediaFacts(
media: readonly ChannelInboundMediaInput[] | null | undefined,
defaults: {
@@ -120,22 +91,10 @@ export function toInboundMediaFacts(
transcribed?: (media: ChannelInboundMediaInput, index: number) => boolean;
} = {},
): InboundMediaFacts[] {
if (!Array.isArray(media)) {
return [];
}
return media.map((entry, index) => ({
path: normalizeString(entry.path),
url: normalizeString(entry.url),
contentType: normalizeString(entry.contentType),
kind: normalizeKind(entry.kind) ?? defaults.kind,
transcribed: entry.transcribed === true || defaults.transcribed?.(entry, index) === true,
messageId: normalizeString(entry.messageId) ?? defaults.messageId,
}));
return normalizeMediaFacts(media, defaults);
}
/**
* Projects inbound attachment facts into transcript history without transient turn-only flags.
*/
/** Projects facts into history without transient turn-only fields. */
export function toHistoryMediaEntries(
media: readonly ChannelInboundMediaInput[] | null | undefined,
defaults: {
@@ -152,23 +111,9 @@ export function toHistoryMediaEntries(
}));
}
/**
* Builds prompt environment media fields while keeping single-item legacy fields populated.
*/
/** Builds the legacy singular/plural environment projection. */
export function buildChannelInboundMediaPayload(
media: readonly InboundMediaFacts[] | null | undefined,
): ChannelInboundMediaPayload {
const entries = Array.isArray(media) ? media : [];
const transcribedIndexes = entries
.map((item, index) => (item.transcribed ? index : undefined))
.filter((index): index is number => index !== undefined);
return {
MediaPath: entries[0]?.path,
MediaUrl: entries[0]?.url ?? entries[0]?.path,
MediaType: entries[0] ? mediaType(entries[0]) : undefined,
MediaPaths: alignedStrings(entries.map((item) => item.path)),
MediaUrls: alignedStrings(entries.map((item) => item.url ?? item.path)),
MediaTypes: alignedStrings(entries.map(mediaType)),
MediaTranscribedIndexes: transcribedIndexes.length > 0 ? transcribedIndexes : undefined,
};
return projectMediaFacts(media);
}
+12 -37
View File
@@ -1,44 +1,19 @@
/**
* Input media item used by channel outbound payload builders.
*/
export type MediaPayloadInput = {
path: string;
contentType?: string;
};
import {
projectMediaFacts,
type MediaFact,
type MediaFactLegacyProjection,
} from "../../media/media-facts.js";
/**
* Legacy-compatible media payload shape consumed by plugin send helpers.
*/
export type MediaPayload = {
MediaPath?: string;
MediaType?: string;
MediaUrl?: string;
MediaPaths?: string[];
MediaUrls?: string[];
MediaTypes?: string[];
};
/** Input media item used by channel outbound payload builders. */
export type MediaPayloadInput = Required<Pick<MediaFact, "path">> & Pick<MediaFact, "contentType">;
/**
* Builds single-item and list media fields for channel outbound helpers.
*/
/** Legacy-compatible media payload shape consumed by plugin send helpers. */
export type MediaPayload = Omit<MediaFactLegacyProjection, "MediaTranscribedIndexes">;
/** Builds single-item and list media fields for channel outbound helpers. */
export function buildMediaPayload(
mediaList: MediaPayloadInput[],
opts?: { preserveMediaTypeCardinality?: boolean },
): MediaPayload {
const first = mediaList[0];
const mediaPaths = mediaList.map((media) => media.path);
const rawMediaTypes = mediaList.map((media) => media.contentType ?? "");
// Some callers need MediaTypes to stay aligned with MediaPaths, including
// blank entries. Others use the compact legacy list of present content types.
const mediaTypes = opts?.preserveMediaTypeCardinality
? rawMediaTypes
: rawMediaTypes.filter((value): value is string => Boolean(value));
return {
MediaPath: first?.path,
MediaType: first?.contentType,
MediaUrl: first?.path,
MediaPaths: mediaPaths.length > 0 ? mediaPaths : undefined,
MediaUrls: mediaPaths.length > 0 ? mediaPaths : undefined,
MediaTypes: mediaTypes.length > 0 ? mediaTypes : undefined,
};
return projectMediaFacts(mediaList, opts?.preserveMediaTypeCardinality ? "aligned" : "compact");
}
+2 -9
View File
@@ -1,4 +1,3 @@
import type { MediaKind } from "@openclaw/media-core/constants";
import type { CommandTurnKind } from "../../auto-reply/command-turn-context.js";
import type {
GetReplyOptions,
@@ -25,6 +24,7 @@ import type {
DurableFinalDeliveryRequirements,
OutboundDeliveryQueuePolicy,
} from "../../infra/outbound/deliver.js";
import type { MediaFact } from "../../media/media-facts.js";
import type { InboundEventKind } from "../inbound-event/kind.js";
import type { CreateChannelReplyPipelineParams } from "../message/reply-pipeline.js";
import type { MessageReceipt } from "../message/types.js";
@@ -135,14 +135,7 @@ export type CommandFacts = {
};
/** Inbound media facts supplied to the agent context. */
export type InboundMediaFacts = {
path?: string;
url?: string;
contentType?: string;
kind?: MediaKind;
transcribed?: boolean;
messageId?: string;
};
export type InboundMediaFacts = Omit<MediaFact, "workspaceDir">;
type MaybePromise<T> = T | Promise<T>;
+104
View File
@@ -0,0 +1,104 @@
import type { MediaKind } from "@openclaw/media-core/constants";
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
/** One ordered runtime attachment; array position is its alignment identity. */
export type MediaFact = {
path?: string;
url?: string;
contentType?: string;
kind?: MediaKind;
transcribed?: boolean;
messageId?: string;
workspaceDir?: string;
};
export type MediaFactInput = {
[Key in keyof MediaFact]?: MediaFact[Key] | null;
};
type MediaFactDefaults<TInput extends MediaFactInput = MediaFactInput> = {
kind?: MediaKind;
messageId?: string;
workspaceDir?: string;
transcribed?: (media: TInput, index: number) => boolean;
};
export type MediaFactLegacyProjection = {
MediaPath?: string;
MediaUrl?: string;
MediaType?: string;
MediaPaths?: string[];
MediaUrls?: string[];
MediaTypes?: string[];
MediaTranscribedIndexes?: number[];
};
function normalizeMediaFact<TInput extends MediaFactInput>(
media: TInput,
index: number,
defaults: MediaFactDefaults<TInput> = {},
): MediaFact {
const workspaceDir = normalizeOptionalString(media.workspaceDir) ?? defaults.workspaceDir;
return {
path: normalizeOptionalString(media.path),
url: normalizeOptionalString(media.url),
contentType: normalizeOptionalString(media.contentType),
kind: media.kind ?? defaults.kind,
transcribed: media.transcribed === true || defaults.transcribed?.(media, index) === true,
messageId: normalizeOptionalString(media.messageId) ?? defaults.messageId,
...(workspaceDir ? { workspaceDir } : {}),
};
}
export function normalizeMediaFacts<TInput extends MediaFactInput>(
media: readonly TInput[] | null | undefined,
defaults: MediaFactDefaults<TInput> = {},
): MediaFact[] {
return Array.isArray(media)
? media.map((entry, index) => normalizeMediaFact(entry, index, defaults))
: [];
}
function projectStrings(
values: Array<string | null | undefined>,
compact: boolean,
preserveEmptyLists: boolean,
): string[] | undefined {
const projected = compact
? values.filter((value): value is string => Boolean(value))
: values.map((value) => value ?? "");
if (projected.length === 0 || (!preserveEmptyLists && !projected.some(Boolean))) {
return undefined;
}
return projected;
}
export function projectMediaFacts(
media: readonly MediaFactInput[] | null | undefined,
mode: "channel" | "compact" | "aligned" = "channel",
): MediaFactLegacyProjection {
const entries = Array.isArray(media) ? media : [];
const preserveEmptyLists = mode !== "channel";
const mediaUrl = (entry: MediaFactInput) =>
(mode === "channel" ? (entry.url ?? entry.path) : entry.path) ?? undefined;
const mediaType = (entry: MediaFactInput) =>
entry.contentType ?? (mode === "channel" ? entry.kind : undefined) ?? undefined;
const transcribedIndexes = entries.flatMap((entry, index) => (entry.transcribed ? [index] : []));
return {
MediaPath: entries[0]?.path ?? undefined,
MediaUrl: entries[0] ? mediaUrl(entries[0]) : undefined,
MediaType: entries[0] ? mediaType(entries[0]) : undefined,
MediaPaths: projectStrings(
entries.map((entry) => entry.path),
false,
preserveEmptyLists,
),
MediaUrls: projectStrings(entries.map(mediaUrl), false, preserveEmptyLists),
MediaTypes: projectStrings(entries.map(mediaType), mode === "compact", preserveEmptyLists),
...(mode !== "channel"
? {}
: {
MediaTranscribedIndexes: transcribedIndexes.length > 0 ? transcribedIndexes : undefined,
}),
};
}
+4 -21
View File
@@ -1,29 +1,12 @@
// Agent media payload exports expose media roots and loaders for plugin-facing agent payloads.
import { projectMediaFacts, type MediaFactLegacyProjection } from "../media/media-facts.js";
export { getAgentScopedMediaLocalRoots } from "../media/local-roots.js";
/** Legacy agent media payload layout consumed by older agent adapters. */
export type AgentMediaPayload = {
MediaPath?: string;
MediaType?: string;
MediaUrl?: string;
MediaPaths?: string[];
MediaUrls?: string[];
MediaTypes?: string[];
};
export type AgentMediaPayload = Omit<MediaFactLegacyProjection, "MediaTranscribedIndexes">;
/** Convert outbound media descriptors into the legacy agent payload field layout. */
export function buildAgentMediaPayload(
mediaList: Array<{ path: string; contentType?: string | null }>,
): AgentMediaPayload {
const first = mediaList[0];
const mediaPaths = mediaList.map((media) => media.path);
const mediaTypes = mediaList.map((media) => media.contentType).filter(Boolean) as string[];
return {
MediaPath: first?.path,
MediaType: first?.contentType ?? undefined,
MediaUrl: first?.path,
MediaPaths: mediaPaths.length > 0 ? mediaPaths : undefined,
MediaUrls: mediaPaths.length > 0 ? mediaPaths : undefined,
MediaTypes: mediaTypes.length > 0 ? mediaTypes : undefined,
};
return projectMediaFacts(mediaList, "compact");
}
+16
View File
@@ -374,6 +374,22 @@ describe("user turn transcript persistence", () => {
});
describe("createUserTurnTranscriptRecorder", () => {
it("accepts and normalizes provider-defined persisted media kinds", () => {
const input: UserTurnInput = {
text: "inspect this attachment",
media: [{ path: " /tmp/provider-media.bin ", kind: " provider/custom-media " }],
};
const recorder = createUserTurnTranscriptRecorder({
input,
target: unusedRecorderTarget,
});
expect(recorder.message).toMatchObject({
MediaPath: "/tmp/provider-media.bin",
MediaType: "provider/custom-media",
});
});
it("persists fallback user turns only once", async () => {
const dir = createTempDir("openclaw-user-turn-recorder-fallback-");
const target = createSqliteTranscriptTarget({ dir });
+2 -4
View File
@@ -4,6 +4,7 @@ import type {
SessionTranscriptTurnExpectedState,
SessionTranscriptTurnLifecyclePatch,
} from "../config/sessions/session-transcript-turn-lifecycle.types.js";
import type { MediaFactInput } from "../media/media-facts.js";
import type { InputProvenance } from "./input-provenance.js";
type UserTurnSessionEntry = {
@@ -13,10 +14,7 @@ type UserTurnSessionEntry = {
threadId?: string | number;
} & Record<string, unknown>;
export type PersistedUserTurnMediaInput = {
path?: string | null;
url?: string | null;
contentType?: string | null;
export type PersistedUserTurnMediaInput = Pick<MediaFactInput, "contentType" | "path" | "url"> & {
kind?: string | null;
};