mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-27 12:56:01 -06:00
fix: keep media transcript text clean
This commit is contained in:
@@ -971,6 +971,111 @@ describe("runPreparedReply media-only handling", () => {
|
||||
expect(call.followupRun.imageOrder).toEqual(["inline"]);
|
||||
});
|
||||
|
||||
it("persists clean media captions instead of model-only media notes", async () => {
|
||||
const tmpDir = await mkdtemp(path.join(os.tmpdir(), "openclaw-followup-image-"));
|
||||
cleanupPaths.push(tmpDir);
|
||||
const imagePath = path.join(tmpDir, "inbound.png");
|
||||
await writeFile(
|
||||
imagePath,
|
||||
Buffer.from(
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=",
|
||||
"base64",
|
||||
),
|
||||
);
|
||||
|
||||
await runPreparedReply(
|
||||
baseParams({
|
||||
ctx: {
|
||||
Body: "What is in this image?",
|
||||
RawBody: "What is in this image?",
|
||||
CommandBody: "What is in this image?",
|
||||
MediaPaths: [imagePath],
|
||||
MediaTypes: ["image/png"],
|
||||
MediaWorkspaceDir: tmpDir,
|
||||
OriginatingChannel: "telegram",
|
||||
OriginatingTo: "42",
|
||||
ChatType: "direct",
|
||||
},
|
||||
sessionCtx: {
|
||||
Body: "[media attached: media://inbound/a.png (image/png)]\nTo send an image back, prefer the message tool (media/path/filePath).\nWhat is in this image?",
|
||||
BodyStripped:
|
||||
"[media attached: media://inbound/a.png (image/png)]\nTo send an image back, prefer the message tool (media/path/filePath).\nWhat is in this image?",
|
||||
Provider: "telegram",
|
||||
OriginatingChannel: "telegram",
|
||||
OriginatingTo: "42",
|
||||
ChatType: "direct",
|
||||
MediaPaths: [imagePath],
|
||||
MediaTypes: ["image/png"],
|
||||
MediaWorkspaceDir: tmpDir,
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const call = requireRunReplyAgentCall();
|
||||
expect(call.followupRun.userMessageForPersistence).toMatchObject({
|
||||
role: "user",
|
||||
content: "What is in this image?",
|
||||
MediaPath: imagePath,
|
||||
MediaPaths: [imagePath],
|
||||
MediaType: "image/png",
|
||||
MediaTypes: ["image/png"],
|
||||
});
|
||||
const persistedContent = call.followupRun.userMessageForPersistence?.content;
|
||||
expect(persistedContent).toBe("What is in this image?");
|
||||
expect(persistedContent).not.toContain("media attached");
|
||||
expect(persistedContent).not.toContain("message tool");
|
||||
});
|
||||
|
||||
it("uses a media-only transcript label for exact media placeholders", async () => {
|
||||
const tmpDir = await mkdtemp(path.join(os.tmpdir(), "openclaw-followup-image-"));
|
||||
cleanupPaths.push(tmpDir);
|
||||
const imagePath = path.join(tmpDir, "inbound.png");
|
||||
await writeFile(
|
||||
imagePath,
|
||||
Buffer.from(
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=",
|
||||
"base64",
|
||||
),
|
||||
);
|
||||
|
||||
await runPreparedReply(
|
||||
baseParams({
|
||||
ctx: {
|
||||
Body: "<media:image>",
|
||||
RawBody: "<media:image>",
|
||||
CommandBody: "<media:image>",
|
||||
MediaPaths: [imagePath],
|
||||
MediaTypes: ["image/png"],
|
||||
MediaWorkspaceDir: tmpDir,
|
||||
OriginatingChannel: "telegram",
|
||||
OriginatingTo: "42",
|
||||
ChatType: "direct",
|
||||
},
|
||||
sessionCtx: {
|
||||
Body: "<media:image>",
|
||||
BodyStripped: "<media:image>",
|
||||
Provider: "telegram",
|
||||
OriginatingChannel: "telegram",
|
||||
OriginatingTo: "42",
|
||||
ChatType: "direct",
|
||||
MediaPaths: [imagePath],
|
||||
MediaTypes: ["image/png"],
|
||||
MediaWorkspaceDir: tmpDir,
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const call = requireRunReplyAgentCall();
|
||||
expect(call.followupRun.userMessageForPersistence).toMatchObject({
|
||||
role: "user",
|
||||
content: "[User sent media without caption]",
|
||||
MediaPath: imagePath,
|
||||
MediaPaths: [imagePath],
|
||||
MediaType: "image/png",
|
||||
MediaTypes: ["image/png"],
|
||||
});
|
||||
});
|
||||
|
||||
it("does not rehydrate current MediaPaths after image understanding enriched the prompt", async () => {
|
||||
const tmpDir = await mkdtemp(path.join(os.tmpdir(), "openclaw-followup-image-"));
|
||||
cleanupPaths.push(tmpDir);
|
||||
|
||||
@@ -35,6 +35,7 @@ import {
|
||||
import {
|
||||
buildPersistedUserTurnMediaInputsFromFields,
|
||||
buildPersistedUserTurnMessage,
|
||||
resolvePersistedUserTurnText,
|
||||
type UserTurnInput,
|
||||
} from "../../sessions/user-turn-transcript.js";
|
||||
import { createLazyImportLoader } from "../../shared/lazy-promise.js";
|
||||
@@ -1134,12 +1135,27 @@ export async function runPreparedReply(
|
||||
? (internalOpts?.queuedFollowupAbortSignal ?? opts?.abortSignal)
|
||||
: undefined;
|
||||
const userTurnMediaForPersistence = buildPersistedUserTurnMediaInputsFromFields(ctx);
|
||||
const userTurnTranscriptText = resolvePersistedUserTurnText(
|
||||
{
|
||||
Transcript: ctx.Transcript,
|
||||
RawBody: ctx.RawBody,
|
||||
CommandBody: ctx.CommandBody,
|
||||
BodyForCommands: ctx.BodyForCommands,
|
||||
Body: ctx.Body,
|
||||
BodyStripped: sessionCtx.BodyStripped,
|
||||
},
|
||||
{
|
||||
hasMedia: userTurnMediaForPersistence.length > 0,
|
||||
fallback: baseBodyTrimmedRaw,
|
||||
},
|
||||
);
|
||||
const userTurnInput =
|
||||
params.userTurnInput ??
|
||||
(userTurnMediaForPersistence.length > 0
|
||||
? {
|
||||
text: baseBodyTrimmedRaw,
|
||||
text: userTurnTranscriptText,
|
||||
media: userTurnMediaForPersistence,
|
||||
mediaOnlyText: "[User sent media without caption]",
|
||||
}
|
||||
: undefined);
|
||||
const userMessageForPersistence = userTurnInput
|
||||
|
||||
@@ -2578,6 +2578,7 @@ export const chatHandlers: GatewayRequestHandlers = {
|
||||
text: parsedMessage,
|
||||
media,
|
||||
timestamp: now,
|
||||
mediaOnlyText: "[User sent media without caption]",
|
||||
}
|
||||
: undefined,
|
||||
);
|
||||
|
||||
@@ -3,6 +3,7 @@ import {
|
||||
buildPersistedUserTurnMediaInputsFromFields,
|
||||
buildPersistedUserTurnMediaFields,
|
||||
buildPersistedUserTurnMessage,
|
||||
resolvePersistedUserTurnText,
|
||||
} from "./user-turn-transcript.js";
|
||||
|
||||
describe("user turn transcript persistence", () => {
|
||||
@@ -180,4 +181,52 @@ describe("user turn transcript persistence", () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolvePersistedUserTurnText", () => {
|
||||
it("prefers clean inbound text over model prompt text", () => {
|
||||
expect(
|
||||
resolvePersistedUserTurnText(
|
||||
{
|
||||
RawBody: "What is in this image?",
|
||||
BodyStripped:
|
||||
"[media attached: media://inbound/a.png]\nTo send an image back, prefer the message tool.\nWhat is in this image?",
|
||||
},
|
||||
{ hasMedia: true },
|
||||
),
|
||||
).toBe("What is in this image?");
|
||||
});
|
||||
|
||||
it("uses audio transcript before media placeholders", () => {
|
||||
expect(
|
||||
resolvePersistedUserTurnText(
|
||||
{
|
||||
Transcript: "please check this voice note",
|
||||
RawBody: "<media:audio>",
|
||||
CommandBody: "<media:audio>",
|
||||
},
|
||||
{ hasMedia: true },
|
||||
),
|
||||
).toBe("please check this voice note");
|
||||
});
|
||||
|
||||
it("ignores exact generated media placeholders only when structured media is present", () => {
|
||||
expect(
|
||||
resolvePersistedUserTurnText(
|
||||
{
|
||||
RawBody: "<media:image> (2 images)",
|
||||
BodyStripped: "<media:image> (2 images)",
|
||||
},
|
||||
{ hasMedia: true, fallback: "fallback" },
|
||||
),
|
||||
).toBe("fallback");
|
||||
expect(
|
||||
resolvePersistedUserTurnText(
|
||||
{
|
||||
RawBody: "<media:image> (2 images)",
|
||||
},
|
||||
{ hasMedia: false },
|
||||
),
|
||||
).toBe("<media:image> (2 images)");
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -26,6 +26,20 @@ export type UserTurnInput = {
|
||||
|
||||
export type BuildPersistedUserTurnMessageParams = UserTurnInput;
|
||||
|
||||
export type PersistedUserTurnTextFieldSource = {
|
||||
Transcript?: string | null;
|
||||
RawBody?: string | null;
|
||||
CommandBody?: string | null;
|
||||
BodyForCommands?: string | null;
|
||||
Body?: string | null;
|
||||
BodyStripped?: string | null;
|
||||
};
|
||||
|
||||
export type ResolvePersistedUserTurnTextOptions = {
|
||||
hasMedia?: boolean;
|
||||
fallback?: string | null;
|
||||
};
|
||||
|
||||
export type PersistedUserTurnMediaFieldSource = {
|
||||
MediaPath?: string | null;
|
||||
MediaPaths?: readonly (string | null | undefined)[] | null;
|
||||
@@ -44,6 +58,45 @@ function normalizeTranscriptText(value: string | null | undefined): string {
|
||||
return value ?? "";
|
||||
}
|
||||
|
||||
const MEDIA_PLACEHOLDER_PATTERN = /^<media:[a-z0-9_-]+>(?:\s+\([^)]*\))?$/i;
|
||||
|
||||
function normalizePersistedUserTextCandidate(
|
||||
value: string | null | undefined,
|
||||
options: { hasMedia: boolean },
|
||||
): string | undefined {
|
||||
const normalized = normalizeOptionalText(value);
|
||||
if (!normalized) {
|
||||
return undefined;
|
||||
}
|
||||
if (options.hasMedia && MEDIA_PLACEHOLDER_PATTERN.test(normalized)) {
|
||||
return undefined;
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
export function resolvePersistedUserTurnText(
|
||||
fields: PersistedUserTurnTextFieldSource | null | undefined,
|
||||
options: ResolvePersistedUserTurnTextOptions = {},
|
||||
): string | undefined {
|
||||
const hasMedia = options.hasMedia === true;
|
||||
const candidates = [
|
||||
fields?.Transcript,
|
||||
fields?.RawBody,
|
||||
fields?.CommandBody,
|
||||
fields?.BodyForCommands,
|
||||
fields?.Body,
|
||||
fields?.BodyStripped,
|
||||
options.fallback,
|
||||
];
|
||||
for (const candidate of candidates) {
|
||||
const normalized = normalizePersistedUserTextCandidate(candidate, { hasMedia });
|
||||
if (normalized) {
|
||||
return normalized;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function mediaTypeForTranscript(media: PersistedUserTurnMediaInput): string {
|
||||
return (
|
||||
normalizeOptionalText(media.contentType) ??
|
||||
|
||||
Reference in New Issue
Block a user