mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
fix(gateway): persist media metadata in agent.request transcripts (#86936)
* fix(gateway): persist agent request transcript media Route inline and offloaded agent.request images through the canonical user-turn transcript recorder across embedded, CLI, and ACP runtimes. Share ordered media persistence with chat.send and cover media-only empty-reply turns. Co-authored-by: Petros Dhespollari <info@peterdsp.dev> * fix(gateway): avoid transcript media shadowing --------- Co-authored-by: Peter Steinberger <steipete@gmail.com>
This commit is contained in:
committed by
GitHub
parent
82b0f7cdc8
commit
6894bb7508
@@ -1401,6 +1401,34 @@ describe("agentCommand – LiveSessionModelSwitchError retry", () => {
|
||||
expect(state.assertLifecycleCurrentMock).toHaveBeenLastCalledWith("test-generation");
|
||||
});
|
||||
|
||||
it("persists structured transcript media for ACP turns", async () => {
|
||||
state.acpResolveSessionMock.mockReturnValue({
|
||||
kind: "ready",
|
||||
meta: {
|
||||
agent: "claude",
|
||||
cwd: "/tmp/workspace",
|
||||
},
|
||||
});
|
||||
|
||||
await agentCommand({
|
||||
message: "[media attached: media://inbound/image-1]",
|
||||
transcriptMessage: "",
|
||||
transcriptMedia: [{ path: "/media/inbound/image-1.png", contentType: "image/png" }],
|
||||
sessionKey: "agent:main:main",
|
||||
});
|
||||
|
||||
expect(state.persistAcpTurnTranscriptMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
transcriptBody: "",
|
||||
userInput: {
|
||||
text: "",
|
||||
media: [{ path: "/media/inbound/image-1.png", contentType: "image/png" }],
|
||||
mediaOnlyText: "[User sent media without caption]",
|
||||
},
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps the initial session touch for local runs", async () => {
|
||||
setupSingleAttemptFallback();
|
||||
state.runAgentAttemptMock.mockResolvedValue(makeSuccessResult("openai", "gpt-5.4"));
|
||||
@@ -3037,6 +3065,33 @@ describe("agentCommand – LiveSessionModelSwitchError retry", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("persists structured transcript media without a caption", async () => {
|
||||
setupSingleAttemptFallback();
|
||||
state.runAgentAttemptMock.mockResolvedValue(makeSuccessResult("openai", "gpt-5.4"));
|
||||
|
||||
await agentCommand({
|
||||
message: "[media attached: media://inbound/image-1]",
|
||||
transcriptMessage: "",
|
||||
transcriptMedia: [{ path: "/media/inbound/image-1.png", contentType: "image/png" }],
|
||||
images: [{ type: "image", data: "aGVsbG8=", mimeType: "image/png" }],
|
||||
to: "+1234567890",
|
||||
});
|
||||
|
||||
const attempt = mockCallArg(state.runAgentAttemptMock) as {
|
||||
suppressPromptPersistenceOnRetry?: boolean;
|
||||
userTurnTranscriptRecorder?: { message?: unknown };
|
||||
};
|
||||
expect(attempt.suppressPromptPersistenceOnRetry).toBe(false);
|
||||
expect(attempt.userTurnTranscriptRecorder?.message).toMatchObject({
|
||||
role: "user",
|
||||
content: "[User sent media without caption]",
|
||||
MediaPath: "/media/inbound/image-1.png",
|
||||
MediaPaths: ["/media/inbound/image-1.png"],
|
||||
MediaType: "image/png",
|
||||
MediaTypes: ["image/png"],
|
||||
});
|
||||
});
|
||||
|
||||
it("propagates non-switch errors without retrying and emits lifecycle error", async () => {
|
||||
state.runWithModelFallbackMock.mockRejectedValueOnce(new Error("provider down"));
|
||||
|
||||
|
||||
@@ -1155,6 +1155,15 @@ async function agentCommandInternal(
|
||||
const transcriptResult = await attemptExecutionRuntime.persistAcpTurnTranscript({
|
||||
body,
|
||||
transcriptBody,
|
||||
...(opts.suppressPromptPersistence !== true && opts.transcriptMedia?.length
|
||||
? {
|
||||
userInput: {
|
||||
text: transcriptBody,
|
||||
media: opts.transcriptMedia,
|
||||
mediaOnlyText: "[User sent media without caption]",
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
finalText: finalTextRaw,
|
||||
sessionId,
|
||||
sessionKey,
|
||||
@@ -1762,12 +1771,25 @@ async function agentCommandInternal(
|
||||
lifecycleEnded: false,
|
||||
};
|
||||
const attemptLifecycleCallbacks = createAgentAttemptLifecycleCallbacks(attemptLifecycleState);
|
||||
const transcriptMedia = opts.transcriptMedia ?? [];
|
||||
const hasTranscriptMedia = transcriptMedia.length > 0;
|
||||
const suppressUserTurnPersistence =
|
||||
opts.suppressPromptPersistence === true || opts.transcriptMessage === "";
|
||||
opts.suppressPromptPersistence === true ||
|
||||
(opts.transcriptMessage === "" && !hasTranscriptMedia);
|
||||
const recorderTranscriptText = transcriptBody || undefined;
|
||||
const userTurnTranscriptRecorder = createUserTurnTranscriptRecorder({
|
||||
...(!suppressUserTurnPersistence && recorderTranscriptText
|
||||
? { input: { text: recorderTranscriptText } }
|
||||
...(!suppressUserTurnPersistence && (recorderTranscriptText || hasTranscriptMedia)
|
||||
? {
|
||||
input: {
|
||||
text: recorderTranscriptText,
|
||||
...(hasTranscriptMedia
|
||||
? {
|
||||
media: transcriptMedia,
|
||||
mediaOnlyText: "[User sent media without caption]",
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
target: {
|
||||
transcriptPath: attemptSessionFile,
|
||||
|
||||
@@ -13,6 +13,7 @@ import { saveAuthProfileStore } from "../auth-profiles/store.js";
|
||||
import type { EmbeddedAgentRunResult } from "../embedded-agent.js";
|
||||
import { FailoverError } from "../failover-error.js";
|
||||
import {
|
||||
persistAcpTurnTranscript,
|
||||
persistCliTurnTranscript,
|
||||
runAgentAttempt as runAgentAttemptImpl,
|
||||
} from "./attempt-execution.js";
|
||||
@@ -1374,6 +1375,46 @@ describe("CLI attempt execution", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("persists a media-only ACP user turn when the reply is empty", async () => {
|
||||
const sessionKey = "agent:main:direct:acp-media-only";
|
||||
const sessionFile = path.join(tmpDir, "session-acp-media-only.jsonl");
|
||||
const sessionEntry: SessionEntry = {
|
||||
sessionId: "session-acp-media-only",
|
||||
sessionFile,
|
||||
updatedAt: Date.now(),
|
||||
};
|
||||
const sessionStore: Record<string, SessionEntry> = { [sessionKey]: sessionEntry };
|
||||
await fs.writeFile(storePath, JSON.stringify(sessionStore, null, 2), "utf-8");
|
||||
|
||||
await persistAcpTurnTranscript({
|
||||
body: "[media attached: media://inbound/image-1]",
|
||||
transcriptBody: "",
|
||||
userInput: {
|
||||
text: "",
|
||||
media: [{ path: "/media/inbound/image-1.png", contentType: "image/png" }],
|
||||
mediaOnlyText: "[User sent media without caption]",
|
||||
},
|
||||
finalText: "",
|
||||
sessionId: sessionEntry.sessionId,
|
||||
sessionKey,
|
||||
sessionEntry,
|
||||
sessionStore,
|
||||
storePath,
|
||||
sessionAgentId: "main",
|
||||
sessionCwd: tmpDir,
|
||||
config: {},
|
||||
});
|
||||
|
||||
expect(await readSessionMessages(sessionFile)).toContainEqual(
|
||||
expect.objectContaining({
|
||||
role: "user",
|
||||
content: "[User sent media without caption]",
|
||||
MediaPath: "/media/inbound/image-1.png",
|
||||
MediaType: "image/png",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("does not append a CLI transcript after the session is deleted", async () => {
|
||||
const sessionKey = "agent:main:subagent:cli-transcript-deleted";
|
||||
const staleSessionFile = path.join(tmpDir, "session-cli-stale.jsonl");
|
||||
|
||||
@@ -29,8 +29,10 @@ import type { PluginMetadataSnapshot } from "../../plugins/plugin-metadata-snaps
|
||||
import { isSubagentSessionKey } from "../../routing/session-key.js";
|
||||
import { annotateInterSessionPromptText } from "../../sessions/input-provenance.js";
|
||||
import {
|
||||
buildPersistedUserTurnMessage,
|
||||
preparePersistedUserTurnMessageForTranscriptWrite,
|
||||
type PersistedUserTurnMessage,
|
||||
type UserTurnInput,
|
||||
type UserTurnTranscriptRecorder,
|
||||
} from "../../sessions/user-turn-transcript.js";
|
||||
import { buildWorkspaceSkillSnapshot } from "../../skills/loading/workspace.js";
|
||||
@@ -308,11 +310,6 @@ async function persistTextTurnTranscript(
|
||||
): Promise<PersistTextTurnTranscriptResult> {
|
||||
const promptText = params.transcriptBody ?? params.body;
|
||||
const replyText = params.finalText;
|
||||
if (!promptText && !replyText) {
|
||||
return { kind: "persisted", sessionEntry: params.sessionEntry };
|
||||
}
|
||||
|
||||
const messages = [];
|
||||
const userMessage =
|
||||
params.userMessage ??
|
||||
(promptText
|
||||
@@ -322,6 +319,11 @@ async function persistTextTurnTranscript(
|
||||
timestamp: Date.now(),
|
||||
} as PersistedUserTurnMessage)
|
||||
: undefined);
|
||||
if (!userMessage && !replyText) {
|
||||
return { kind: "persisted", sessionEntry: params.sessionEntry };
|
||||
}
|
||||
|
||||
const messages = [];
|
||||
if (userMessage) {
|
||||
messages.push({
|
||||
message: userMessage,
|
||||
@@ -405,6 +407,7 @@ function isClaudeCliProvider(provider: string): boolean {
|
||||
export async function persistAcpTurnTranscript(params: {
|
||||
body: string;
|
||||
transcriptBody?: string;
|
||||
userInput?: UserTurnInput;
|
||||
finalText: string;
|
||||
sessionId: string;
|
||||
sessionKey: string;
|
||||
@@ -418,6 +421,7 @@ export async function persistAcpTurnTranscript(params: {
|
||||
}): Promise<PersistTextTurnTranscriptResult> {
|
||||
return await persistTextTurnTranscript({
|
||||
...params,
|
||||
...(params.userInput ? { userMessage: buildPersistedUserTurnMessage(params.userInput) } : {}),
|
||||
assistant: {
|
||||
api: "openai-responses",
|
||||
provider: "openclaw",
|
||||
|
||||
@@ -10,6 +10,7 @@ import type { ChannelOutboundTargetMode } from "../../channels/plugins/types.pub
|
||||
import type { PromptImageOrderEntry } from "../../media/prompt-image-order.js";
|
||||
import type { PluginHookChannelContext } from "../../plugins/hook-types.js";
|
||||
import type { InputProvenance } from "../../sessions/input-provenance.js";
|
||||
import type { UserTurnInput } from "../../sessions/user-turn-transcript.js";
|
||||
import type { ExecElevatedDefaults } from "../bash-tools.exec-types.js";
|
||||
import type { BootstrapContextRunKind } from "../bootstrap-mode.js";
|
||||
import type { AgentStreamParams, ClientToolDefinition } from "./shared-types.js";
|
||||
@@ -58,6 +59,8 @@ export type AgentCommandOpts = {
|
||||
message: string;
|
||||
/** User-visible transcript body; defaults to message and excludes runtime-only context. */
|
||||
transcriptMessage?: string;
|
||||
/** Durable media metadata for the user-visible transcript turn. */
|
||||
transcriptMedia?: UserTurnInput["media"];
|
||||
/** Optional image attachments for multimodal messages. */
|
||||
images?: ImageContent[];
|
||||
/** Original inline/offloaded attachment order for inbound images. */
|
||||
|
||||
@@ -29,6 +29,7 @@ import {
|
||||
type ChatAttachment,
|
||||
DEFAULT_CHAT_ATTACHMENT_MAX_MB,
|
||||
parseMessageWithAttachments,
|
||||
persistInboundImagesForTranscript,
|
||||
resolveChatAttachmentMaxBytes,
|
||||
UnsupportedAttachmentError,
|
||||
} from "./chat-attachments.js";
|
||||
@@ -134,6 +135,48 @@ afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe("persistInboundImagesForTranscript", () => {
|
||||
it("preserves mixed image order and appends non-image offloads", async () => {
|
||||
saveMediaBufferMock.mockResolvedValueOnce({
|
||||
id: "inline",
|
||||
path: "/media/inbound/inline.jpg",
|
||||
size: 5,
|
||||
contentType: "image/jpeg",
|
||||
});
|
||||
|
||||
const saved = await persistInboundImagesForTranscript({
|
||||
images: [{ type: "image", data: "aGVsbG8=", mimeType: "image/jpeg" }],
|
||||
imageOrder: ["offloaded", "inline"],
|
||||
offloadedRefs: [
|
||||
{
|
||||
mediaRef: "media://inbound/offloaded",
|
||||
id: "offloaded",
|
||||
path: "/media/inbound/offloaded.png",
|
||||
mimeType: "image/png",
|
||||
label: "offloaded.png",
|
||||
sizeBytes: 2_100_000,
|
||||
},
|
||||
{
|
||||
mediaRef: "media://inbound/report",
|
||||
id: "report",
|
||||
path: "/media/inbound/report.pdf",
|
||||
mimeType: "application/pdf",
|
||||
label: "report.pdf",
|
||||
sizeBytes: 100,
|
||||
},
|
||||
],
|
||||
log: { warn: vi.fn() },
|
||||
logContext: "test",
|
||||
});
|
||||
|
||||
expect(saved.map((entry) => entry.path)).toEqual([
|
||||
"/media/inbound/offloaded.png",
|
||||
"/media/inbound/inline.jpg",
|
||||
"/media/inbound/report.pdf",
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseMessageWithAttachments", () => {
|
||||
it("strips data URL prefix", async () => {
|
||||
const parsed = await parseMessageWithAttachments(
|
||||
|
||||
@@ -8,7 +8,7 @@ import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import { formatErrorMessage } from "../infra/errors.js";
|
||||
import type { PromptImageOrderEntry } from "../media/prompt-image-order.js";
|
||||
import { sniffMimeFromBase64 } from "../media/sniff-mime-from-base64.js";
|
||||
import { deleteMediaBuffer, saveMediaBuffer } from "../media/store.js";
|
||||
import { deleteMediaBuffer, saveMediaBuffer, type SavedMedia } from "../media/store.js";
|
||||
|
||||
export type ChatAttachment = {
|
||||
type?: string;
|
||||
@@ -50,7 +50,7 @@ type NormalizedAttachment = {
|
||||
base64: string;
|
||||
};
|
||||
|
||||
type SavedMedia = {
|
||||
type SavedMediaRef = {
|
||||
id: string;
|
||||
path: string;
|
||||
};
|
||||
@@ -60,6 +60,58 @@ const TEXT_ONLY_OFFLOAD_LIMIT = 10;
|
||||
|
||||
export const DEFAULT_CHAT_ATTACHMENT_MAX_MB = 20;
|
||||
|
||||
export async function persistInboundImagesForTranscript(params: {
|
||||
images: ChatImageContent[];
|
||||
imageOrder: PromptImageOrderEntry[];
|
||||
offloadedRefs: OffloadedRef[];
|
||||
log: Pick<AttachmentLog, "warn">;
|
||||
logContext: string;
|
||||
}): Promise<SavedMedia[]> {
|
||||
const inline: SavedMedia[] = [];
|
||||
for (const image of params.images) {
|
||||
try {
|
||||
inline.push(
|
||||
await saveMediaBuffer(Buffer.from(image.data, "base64"), image.mimeType, "inbound"),
|
||||
);
|
||||
} catch (err) {
|
||||
params.log.warn(
|
||||
`${params.logContext}: failed to persist inbound image (${image.mimeType}): ${formatErrorMessage(err)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const imageOffloaded: SavedMedia[] = [];
|
||||
const nonImageOffloaded: SavedMedia[] = [];
|
||||
for (const ref of params.offloadedRefs) {
|
||||
const saved = {
|
||||
id: ref.id,
|
||||
path: ref.path,
|
||||
size: ref.sizeBytes,
|
||||
contentType: ref.mimeType,
|
||||
};
|
||||
(ref.mimeType.startsWith("image/") ? imageOffloaded : nonImageOffloaded).push(saved);
|
||||
}
|
||||
if (params.imageOrder.length === 0) {
|
||||
return [...inline, ...imageOffloaded, ...nonImageOffloaded];
|
||||
}
|
||||
|
||||
const ordered: SavedMedia[] = [];
|
||||
let inlineIndex = 0;
|
||||
let offloadedIndex = 0;
|
||||
for (const entry of params.imageOrder) {
|
||||
const media = entry === "inline" ? inline[inlineIndex++] : imageOffloaded[offloadedIndex++];
|
||||
if (media) {
|
||||
ordered.push(media);
|
||||
}
|
||||
}
|
||||
ordered.push(
|
||||
...inline.slice(inlineIndex),
|
||||
...imageOffloaded.slice(offloadedIndex),
|
||||
...nonImageOffloaded,
|
||||
);
|
||||
return ordered;
|
||||
}
|
||||
|
||||
/** Resolve the maximum decoded attachment size accepted for chat image inputs. */
|
||||
export function resolveChatAttachmentMaxBytes(cfg: OpenClawConfig): number {
|
||||
const configured = cfg.agents?.defaults?.mediaMaxMb;
|
||||
@@ -197,7 +249,7 @@ function ensureExtension(label: string, mime: string): string {
|
||||
return ext ? `${label}${ext}` : label;
|
||||
}
|
||||
|
||||
function assertSavedMedia(value: unknown, label: string): SavedMedia {
|
||||
function assertSavedMedia(value: unknown, label: string): SavedMediaRef {
|
||||
if (
|
||||
value === null ||
|
||||
typeof value !== "object" ||
|
||||
@@ -382,7 +434,7 @@ export async function parseMessageWithAttachments(
|
||||
const buffer = Buffer.from(b64, "base64");
|
||||
verifyDecodedSize(buffer, sizeBytes, label);
|
||||
|
||||
let savedMedia: SavedMedia;
|
||||
let savedMedia: SavedMediaRef;
|
||||
try {
|
||||
const labelWithExt = ensureExtension(label, finalMime);
|
||||
const rawResult = await saveMediaBuffer(
|
||||
|
||||
@@ -99,12 +99,7 @@ import { parseInboundMediaUri } from "../../media/media-reference.js";
|
||||
import type { PromptImageOrderEntry } from "../../media/prompt-image-order.js";
|
||||
import { renderQrPngDataUrl } from "../../media/qr-image.js";
|
||||
import { renderQrTerminal } from "../../media/qr-terminal.js";
|
||||
import {
|
||||
deleteMediaBuffer,
|
||||
MEDIA_MAX_BYTES,
|
||||
type SavedMedia,
|
||||
saveMediaBuffer,
|
||||
} from "../../media/store.js";
|
||||
import { deleteMediaBuffer, MEDIA_MAX_BYTES, type SavedMedia } from "../../media/store.js";
|
||||
import { createChannelMessageReplyPipeline } from "../../plugin-sdk/channel-outbound.js";
|
||||
import type { ChannelRouteRef } from "../../plugin-sdk/channel-route.js";
|
||||
import { isPluginOwnedSessionBindingRecord } from "../../plugins/conversation-binding.js";
|
||||
@@ -148,6 +143,7 @@ import {
|
||||
MediaOffloadError,
|
||||
type OffloadedRef,
|
||||
parseMessageWithAttachments,
|
||||
persistInboundImagesForTranscript,
|
||||
resolveChatAttachmentMaxBytes,
|
||||
UnsupportedAttachmentError,
|
||||
} from "../chat-attachments.js";
|
||||
@@ -1333,73 +1329,13 @@ async function persistChatSendImages(params: {
|
||||
) {
|
||||
return [];
|
||||
}
|
||||
const inlineSaved: SavedMedia[] = [];
|
||||
for (const img of params.images) {
|
||||
try {
|
||||
inlineSaved.push(
|
||||
await saveMediaBuffer(Buffer.from(img.data, "base64"), img.mimeType, "inbound"),
|
||||
);
|
||||
} catch (err) {
|
||||
params.logGateway.warn(
|
||||
`chat.send: failed to persist inbound image (${img.mimeType}): ${formatForLog(err)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
// imageOrder now only tracks image slots (see chat-attachments.ts), so split
|
||||
// offloaded refs by mime: image offloads interleave with inline images via
|
||||
// imageOrder, and non-image offloads append to the transcript tail. Without
|
||||
// this split a non-image file would consume the next image slot whenever
|
||||
// both kinds appear in the same request.
|
||||
const imageOffloadedSaved: SavedMedia[] = [];
|
||||
const nonImageOffloadedSaved: SavedMedia[] = [];
|
||||
for (const ref of params.offloadedRefs) {
|
||||
const entry: SavedMedia = {
|
||||
id: ref.id,
|
||||
path: ref.path,
|
||||
size: 0,
|
||||
contentType: ref.mimeType,
|
||||
};
|
||||
if (ref.mimeType.startsWith("image/")) {
|
||||
imageOffloadedSaved.push(entry);
|
||||
} else {
|
||||
nonImageOffloadedSaved.push(entry);
|
||||
}
|
||||
}
|
||||
if (params.imageOrder.length === 0) {
|
||||
return [...inlineSaved, ...imageOffloadedSaved, ...nonImageOffloadedSaved];
|
||||
}
|
||||
const saved: SavedMedia[] = [];
|
||||
let inlineIndex = 0;
|
||||
let offloadedIndex = 0;
|
||||
for (const entry of params.imageOrder) {
|
||||
if (entry === "inline") {
|
||||
const inline = inlineSaved[inlineIndex++];
|
||||
if (inline) {
|
||||
saved.push(inline);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const offloaded = imageOffloadedSaved[offloadedIndex++];
|
||||
if (offloaded) {
|
||||
saved.push(offloaded);
|
||||
}
|
||||
}
|
||||
for (; inlineIndex < inlineSaved.length; inlineIndex++) {
|
||||
const inline = inlineSaved[inlineIndex];
|
||||
if (inline) {
|
||||
saved.push(inline);
|
||||
}
|
||||
}
|
||||
for (; offloadedIndex < imageOffloadedSaved.length; offloadedIndex++) {
|
||||
const offloaded = imageOffloadedSaved[offloadedIndex];
|
||||
if (offloaded) {
|
||||
saved.push(offloaded);
|
||||
}
|
||||
}
|
||||
for (const offloaded of nonImageOffloadedSaved) {
|
||||
saved.push(offloaded);
|
||||
}
|
||||
return saved;
|
||||
return await persistInboundImagesForTranscript({
|
||||
images: params.images,
|
||||
imageOrder: params.imageOrder,
|
||||
offloadedRefs: params.offloadedRefs,
|
||||
log: params.logGateway,
|
||||
logContext: "chat.send",
|
||||
});
|
||||
}
|
||||
|
||||
function stripTrailingOffloadedMediaMarkers(message: string, refs: OffloadedRef[]): string {
|
||||
|
||||
@@ -18,7 +18,11 @@ export { enqueueSystemEvent } from "../infra/system-events.js";
|
||||
export { deleteMediaBuffer } from "../media/store.js";
|
||||
export { normalizeMainKey, scopedHeartbeatWakeOptions } from "../routing/session-key.js";
|
||||
export { defaultRuntime } from "../runtime.js";
|
||||
export { parseMessageWithAttachments, resolveChatAttachmentMaxBytes } from "./chat-attachments.js";
|
||||
export {
|
||||
parseMessageWithAttachments,
|
||||
persistInboundImagesForTranscript,
|
||||
resolveChatAttachmentMaxBytes,
|
||||
} from "./chat-attachments.js";
|
||||
export { normalizeRpcAttachmentsToChatAttachments } from "./server-methods/attachment-normalize.js";
|
||||
export {
|
||||
loadSessionEntry,
|
||||
|
||||
@@ -54,6 +54,7 @@ const loadOrCreateProcessDeviceIdentityMock = vi.hoisted(() =>
|
||||
})),
|
||||
);
|
||||
const parseMessageWithAttachmentsMock = vi.hoisted(() => vi.fn());
|
||||
const persistInboundImagesForTranscriptMock = vi.hoisted(() => vi.fn());
|
||||
const normalizeChannelIdMock = vi.hoisted(() =>
|
||||
vi.fn((channel?: string | null) => channel ?? null),
|
||||
);
|
||||
@@ -123,6 +124,7 @@ const runtimeMocks = vi.hoisted(() => ({
|
||||
model: entry?.model ?? "default-model",
|
||||
}),
|
||||
),
|
||||
persistInboundImagesForTranscript: persistInboundImagesForTranscriptMock,
|
||||
sanitizeInboundSystemTags: sanitizeInboundSystemTagsMock,
|
||||
scopedHeartbeatWakeOptions: vi.fn((sessionKey?: string, opts?: { reason: string }) => {
|
||||
const wakeOptions = { reason: opts?.reason };
|
||||
@@ -267,6 +269,8 @@ describe("node exec events", () => {
|
||||
registerApnsRegistrationVi.mockClear();
|
||||
loadOrCreateProcessDeviceIdentityMock.mockClear();
|
||||
normalizeChannelIdVi.mockClear();
|
||||
persistInboundImagesForTranscriptMock.mockReset();
|
||||
persistInboundImagesForTranscriptMock.mockResolvedValue([]);
|
||||
normalizeChannelIdVi.mockImplementation((channel?: string | null) => channel ?? null);
|
||||
sanitizeInboundSystemTagsMock.mockClear();
|
||||
updatePairedDeviceMetadataMock.mockClear();
|
||||
@@ -1273,6 +1277,60 @@ describe("agent request events", () => {
|
||||
expectFields(parseCall?.[2], { supportsInlineImages: false });
|
||||
});
|
||||
|
||||
it("passes ordered durable media metadata to the agent transcript recorder", async () => {
|
||||
parseMessageWithAttachmentsMock.mockResolvedValueOnce({
|
||||
message: "describe\n[media attached: media://inbound/offloaded]",
|
||||
images: [{ type: "image", data: "aGVsbG8=", mimeType: "image/jpeg" }],
|
||||
imageOrder: ["offloaded", "inline"],
|
||||
offloadedRefs: [
|
||||
{
|
||||
mediaRef: "media://inbound/offloaded",
|
||||
id: "offloaded",
|
||||
path: "/media/inbound/offloaded.png",
|
||||
mimeType: "image/png",
|
||||
label: "offloaded.png",
|
||||
sizeBytes: 2_100_000,
|
||||
},
|
||||
],
|
||||
});
|
||||
persistInboundImagesForTranscriptMock.mockResolvedValueOnce([
|
||||
{
|
||||
id: "offloaded",
|
||||
path: "/media/inbound/offloaded.png",
|
||||
size: 2_100_000,
|
||||
contentType: "image/png",
|
||||
},
|
||||
{
|
||||
id: "saved-inline",
|
||||
path: "/media/inbound/saved-inline.jpg",
|
||||
size: 5,
|
||||
contentType: "image/jpeg",
|
||||
},
|
||||
]);
|
||||
|
||||
await handleNodeEvent(buildCtx(), "node-media", {
|
||||
event: "agent.request",
|
||||
payloadJSON: JSON.stringify({
|
||||
message: "describe",
|
||||
sessionKey: "agent:main:main",
|
||||
attachments: [{ type: "image", mimeType: "image/png", content: "AAAA" }],
|
||||
}),
|
||||
});
|
||||
|
||||
expect(persistInboundImagesForTranscriptMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ imageOrder: ["offloaded", "inline"] }),
|
||||
);
|
||||
expect(agentCommandMock).toHaveBeenCalledTimes(1);
|
||||
expectFields(mockCallArg(agentCommandMock), {
|
||||
message: "describe\n[media attached: media://inbound/offloaded]",
|
||||
transcriptMessage: "describe",
|
||||
transcriptMedia: [
|
||||
{ path: "/media/inbound/offloaded.png", contentType: "image/png" },
|
||||
{ path: "/media/inbound/saved-inline.jpg", contentType: "image/jpeg" },
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("declines non-image attachments cleanly when parse throws UnsupportedAttachmentError", async () => {
|
||||
const warn = vi.fn();
|
||||
const ctx = buildCtx();
|
||||
|
||||
@@ -43,6 +43,7 @@ import {
|
||||
resolveOutboundTarget,
|
||||
resolveSessionAgentId,
|
||||
resolveSessionModelRef,
|
||||
persistInboundImagesForTranscript,
|
||||
sanitizeInboundSystemTags,
|
||||
sendDurableMessageBatch,
|
||||
canonicalizeSessionEntryAliases,
|
||||
@@ -478,11 +479,14 @@ export const handleNodeEvent = async (
|
||||
const { storePath, entry, canonicalKey, storeKeys } = loadSessionEntry(sessionKey);
|
||||
|
||||
let message = (link?.message ?? "").trim();
|
||||
const transcriptMessage = message;
|
||||
const normalizedAttachments = normalizeRpcAttachmentsToChatAttachments(
|
||||
link?.attachments ?? undefined,
|
||||
);
|
||||
let images: Array<{ type: "image"; data: string; mimeType: string }> = [];
|
||||
let imageOrder: PromptImageOrderEntry[] = [];
|
||||
let offloadedRefs: Awaited<ReturnType<typeof parseMessageWithAttachments>>["offloadedRefs"] =
|
||||
[];
|
||||
if (!message && normalizedAttachments.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
@@ -510,6 +514,7 @@ export const handleNodeEvent = async (
|
||||
message = parsed.message.trim();
|
||||
images = parsed.images;
|
||||
imageOrder = parsed.imageOrder;
|
||||
offloadedRefs = parsed.offloadedRefs;
|
||||
if (message.length > 20_000) {
|
||||
ctx.logGateway.warn(
|
||||
`agent.request message exceeds limit after attachment parsing (length=${message.length})`,
|
||||
@@ -596,11 +601,22 @@ export const handleNodeEvent = async (
|
||||
);
|
||||
}
|
||||
|
||||
const transcriptMedia = (
|
||||
await persistInboundImagesForTranscript({
|
||||
images,
|
||||
imageOrder,
|
||||
offloadedRefs,
|
||||
log: ctx.logGateway,
|
||||
logContext: "agent.request",
|
||||
})
|
||||
).map((media) => ({ path: media.path, contentType: media.contentType }));
|
||||
|
||||
dispatchNodeAgentCommand(ctx, nodeId, {
|
||||
runId: sessionId,
|
||||
message,
|
||||
images,
|
||||
imageOrder,
|
||||
...(transcriptMedia.length > 0 ? { transcriptMessage, transcriptMedia } : {}),
|
||||
sessionId,
|
||||
sessionKey: canonicalKey,
|
||||
thinking: link?.thinking ?? undefined,
|
||||
|
||||
@@ -258,7 +258,7 @@ function readOpenClawMessageMeta(message: AgentMessage): Record<string, unknown>
|
||||
: undefined;
|
||||
}
|
||||
|
||||
function buildPersistedUserTurnMessage(params: UserTurnInput): PersistedUserTurnMessage {
|
||||
export function buildPersistedUserTurnMessage(params: UserTurnInput): PersistedUserTurnMessage {
|
||||
const mediaFields = buildPersistedUserTurnMediaFields(params.media);
|
||||
const hasMedia = Boolean(mediaFields.MediaPath);
|
||||
const text = normalizeTranscriptText(params.text);
|
||||
|
||||
Reference in New Issue
Block a user