fix: keep media transcript text clean

This commit is contained in:
Shakker
2026-05-25 15:56:37 +01:00
committed by Shakker
parent 928a75a365
commit 91aee9cd51
5 changed files with 225 additions and 1 deletions
@@ -971,6 +971,111 @@ describe("runPreparedReply media-only handling", () => {
expect(call.followupRun.imageOrder).toEqual(["inline"]);
});
it("persists clean media captions instead of model-only media notes", async () => {
const tmpDir = await mkdtemp(path.join(os.tmpdir(), "openclaw-followup-image-"));
cleanupPaths.push(tmpDir);
const imagePath = path.join(tmpDir, "inbound.png");
await writeFile(
imagePath,
Buffer.from(
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=",
"base64",
),
);
await runPreparedReply(
baseParams({
ctx: {
Body: "What is in this image?",
RawBody: "What is in this image?",
CommandBody: "What is in this image?",
MediaPaths: [imagePath],
MediaTypes: ["image/png"],
MediaWorkspaceDir: tmpDir,
OriginatingChannel: "telegram",
OriginatingTo: "42",
ChatType: "direct",
},
sessionCtx: {
Body: "[media attached: media://inbound/a.png (image/png)]\nTo send an image back, prefer the message tool (media/path/filePath).\nWhat is in this image?",
BodyStripped:
"[media attached: media://inbound/a.png (image/png)]\nTo send an image back, prefer the message tool (media/path/filePath).\nWhat is in this image?",
Provider: "telegram",
OriginatingChannel: "telegram",
OriginatingTo: "42",
ChatType: "direct",
MediaPaths: [imagePath],
MediaTypes: ["image/png"],
MediaWorkspaceDir: tmpDir,
},
}),
);
const call = requireRunReplyAgentCall();
expect(call.followupRun.userMessageForPersistence).toMatchObject({
role: "user",
content: "What is in this image?",
MediaPath: imagePath,
MediaPaths: [imagePath],
MediaType: "image/png",
MediaTypes: ["image/png"],
});
const persistedContent = call.followupRun.userMessageForPersistence?.content;
expect(persistedContent).toBe("What is in this image?");
expect(persistedContent).not.toContain("media attached");
expect(persistedContent).not.toContain("message tool");
});
it("uses a media-only transcript label for exact media placeholders", async () => {
const tmpDir = await mkdtemp(path.join(os.tmpdir(), "openclaw-followup-image-"));
cleanupPaths.push(tmpDir);
const imagePath = path.join(tmpDir, "inbound.png");
await writeFile(
imagePath,
Buffer.from(
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=",
"base64",
),
);
await runPreparedReply(
baseParams({
ctx: {
Body: "<media:image>",
RawBody: "<media:image>",
CommandBody: "<media:image>",
MediaPaths: [imagePath],
MediaTypes: ["image/png"],
MediaWorkspaceDir: tmpDir,
OriginatingChannel: "telegram",
OriginatingTo: "42",
ChatType: "direct",
},
sessionCtx: {
Body: "<media:image>",
BodyStripped: "<media:image>",
Provider: "telegram",
OriginatingChannel: "telegram",
OriginatingTo: "42",
ChatType: "direct",
MediaPaths: [imagePath],
MediaTypes: ["image/png"],
MediaWorkspaceDir: tmpDir,
},
}),
);
const call = requireRunReplyAgentCall();
expect(call.followupRun.userMessageForPersistence).toMatchObject({
role: "user",
content: "[User sent media without caption]",
MediaPath: imagePath,
MediaPaths: [imagePath],
MediaType: "image/png",
MediaTypes: ["image/png"],
});
});
it("does not rehydrate current MediaPaths after image understanding enriched the prompt", async () => {
const tmpDir = await mkdtemp(path.join(os.tmpdir(), "openclaw-followup-image-"));
cleanupPaths.push(tmpDir);
+17 -1
View File
@@ -35,6 +35,7 @@ import {
import {
buildPersistedUserTurnMediaInputsFromFields,
buildPersistedUserTurnMessage,
resolvePersistedUserTurnText,
type UserTurnInput,
} from "../../sessions/user-turn-transcript.js";
import { createLazyImportLoader } from "../../shared/lazy-promise.js";
@@ -1134,12 +1135,27 @@ export async function runPreparedReply(
? (internalOpts?.queuedFollowupAbortSignal ?? opts?.abortSignal)
: undefined;
const userTurnMediaForPersistence = buildPersistedUserTurnMediaInputsFromFields(ctx);
const userTurnTranscriptText = resolvePersistedUserTurnText(
{
Transcript: ctx.Transcript,
RawBody: ctx.RawBody,
CommandBody: ctx.CommandBody,
BodyForCommands: ctx.BodyForCommands,
Body: ctx.Body,
BodyStripped: sessionCtx.BodyStripped,
},
{
hasMedia: userTurnMediaForPersistence.length > 0,
fallback: baseBodyTrimmedRaw,
},
);
const userTurnInput =
params.userTurnInput ??
(userTurnMediaForPersistence.length > 0
? {
text: baseBodyTrimmedRaw,
text: userTurnTranscriptText,
media: userTurnMediaForPersistence,
mediaOnlyText: "[User sent media without caption]",
}
: undefined);
const userMessageForPersistence = userTurnInput
+1
View File
@@ -2578,6 +2578,7 @@ export const chatHandlers: GatewayRequestHandlers = {
text: parsedMessage,
media,
timestamp: now,
mediaOnlyText: "[User sent media without caption]",
}
: undefined,
);
+49
View File
@@ -3,6 +3,7 @@ import {
buildPersistedUserTurnMediaInputsFromFields,
buildPersistedUserTurnMediaFields,
buildPersistedUserTurnMessage,
resolvePersistedUserTurnText,
} from "./user-turn-transcript.js";
describe("user turn transcript persistence", () => {
@@ -180,4 +181,52 @@ describe("user turn transcript persistence", () => {
});
});
});
describe("resolvePersistedUserTurnText", () => {
it("prefers clean inbound text over model prompt text", () => {
expect(
resolvePersistedUserTurnText(
{
RawBody: "What is in this image?",
BodyStripped:
"[media attached: media://inbound/a.png]\nTo send an image back, prefer the message tool.\nWhat is in this image?",
},
{ hasMedia: true },
),
).toBe("What is in this image?");
});
it("uses audio transcript before media placeholders", () => {
expect(
resolvePersistedUserTurnText(
{
Transcript: "please check this voice note",
RawBody: "<media:audio>",
CommandBody: "<media:audio>",
},
{ hasMedia: true },
),
).toBe("please check this voice note");
});
it("ignores exact generated media placeholders only when structured media is present", () => {
expect(
resolvePersistedUserTurnText(
{
RawBody: "<media:image> (2 images)",
BodyStripped: "<media:image> (2 images)",
},
{ hasMedia: true, fallback: "fallback" },
),
).toBe("fallback");
expect(
resolvePersistedUserTurnText(
{
RawBody: "<media:image> (2 images)",
},
{ hasMedia: false },
),
).toBe("<media:image> (2 images)");
});
});
});
+53
View File
@@ -26,6 +26,20 @@ export type UserTurnInput = {
export type BuildPersistedUserTurnMessageParams = UserTurnInput;
export type PersistedUserTurnTextFieldSource = {
Transcript?: string | null;
RawBody?: string | null;
CommandBody?: string | null;
BodyForCommands?: string | null;
Body?: string | null;
BodyStripped?: string | null;
};
export type ResolvePersistedUserTurnTextOptions = {
hasMedia?: boolean;
fallback?: string | null;
};
export type PersistedUserTurnMediaFieldSource = {
MediaPath?: string | null;
MediaPaths?: readonly (string | null | undefined)[] | null;
@@ -44,6 +58,45 @@ function normalizeTranscriptText(value: string | null | undefined): string {
return value ?? "";
}
const MEDIA_PLACEHOLDER_PATTERN = /^<media:[a-z0-9_-]+>(?:\s+\([^)]*\))?$/i;
function normalizePersistedUserTextCandidate(
value: string | null | undefined,
options: { hasMedia: boolean },
): string | undefined {
const normalized = normalizeOptionalText(value);
if (!normalized) {
return undefined;
}
if (options.hasMedia && MEDIA_PLACEHOLDER_PATTERN.test(normalized)) {
return undefined;
}
return normalized;
}
export function resolvePersistedUserTurnText(
fields: PersistedUserTurnTextFieldSource | null | undefined,
options: ResolvePersistedUserTurnTextOptions = {},
): string | undefined {
const hasMedia = options.hasMedia === true;
const candidates = [
fields?.Transcript,
fields?.RawBody,
fields?.CommandBody,
fields?.BodyForCommands,
fields?.Body,
fields?.BodyStripped,
options.fallback,
];
for (const candidate of candidates) {
const normalized = normalizePersistedUserTextCandidate(candidate, { hasMedia });
if (normalized) {
return normalized;
}
}
return undefined;
}
function mediaTypeForTranscript(media: PersistedUserTurnMediaInput): string {
return (
normalizeOptionalText(media.contentType) ??