diff --git a/src/auto-reply/reply/dispatch-acp.test.ts b/src/auto-reply/reply/dispatch-acp.test.ts index 9fd2362e3e3d..a8fb1c54b99d 100644 --- a/src/auto-reply/reply/dispatch-acp.test.ts +++ b/src/auto-reply/reply/dispatch-acp.test.ts @@ -1216,24 +1216,36 @@ describe("tryDispatchAcpReply", () => { path: "/tmp/recent-2.png", contentType: "image/png", sender: "Recent 2", + sentAtMs: 1_699_999_997_000, + messagePosition: 6, + messageCount: 9, messageId: "recent-2", }, { path: "/tmp/recent-3.png", contentType: "image/png", sender: "Recent 3", + sentAtMs: 1_699_999_998_000, + messagePosition: 7, + messageCount: 9, messageId: "recent-3", }, { path: "/tmp/recent-4.png", contentType: "image/png", sender: "Recent 4", + sentAtMs: 1_699_999_999_000, + messagePosition: 8, + messageCount: 9, messageId: "recent-4", }, { path: "C:\\Users\\Alice\\Pictures\\recent.png", contentType: "image/png", sender: "Windows", + sentAtMs: 1_699_999_999_500, + messagePosition: 9, + messageCount: 9, messageId: "windows", }, ]); @@ -1247,6 +1259,9 @@ describe("tryDispatchAcpReply", () => { path: "/tmp/secret.png", contentType: "image/png", sender: "@alice", + sentAtMs: 1_700_000_000_000, + messagePosition: 2, + messageCount: 5, messageId: "msg-1", }, ], @@ -1254,6 +1269,8 @@ describe("tryDispatchAcpReply", () => { expect(text).toContain("what is this?"); expect(text).toContain("Recent image 1 from @alice, message msg-1"); + expect(text).toContain("sent at 2023-11-14T22:13:20.000Z"); + expect(text).toContain("message 2 of 5 in available history"); expect(text).not.toContain("/tmp/secret.png"); }); @@ -1309,6 +1326,9 @@ describe("tryDispatchAcpReply", () => { path: imagePath, contentType: "image/png", sender: "@alice", + sentAtMs: 1_700_000_000_000, + messagePosition: 1, + messageCount: 1, messageId: "msg-1", }, ]); @@ -1557,6 +1577,9 @@ describe("tryDispatchAcpReply", () => { path: historyPath, contentType: "image/png", sender: "@alice", + sentAtMs: 1_700_000_000_000, + messagePosition: 1, + messageCount: 1, messageId: "msg-history", }, ]); @@ -1593,6 +1616,54 @@ describe("tryDispatchAcpReply", () => { ]); }); + it("annotates recent history images with sent time and available history position", async () => { + setReadyAcpResolution(); + const historyPath = "/tmp/openclaw-history-metadata.png"; + const historyImage = Buffer.from("history-image"); + acpAttachmentBuffers.set(historyPath, historyImage); + + await runDispatch({ + bodyForAgent: "describe current state", + ctxOverrides: { + Timestamp: 1_700_000_060_000, + InboundHistory: [ + { + sender: "@alice", + body: "bug report", + timestamp: 1_699_999_980_000, + messageId: "msg-before", + }, + { + sender: "@bob", + body: "", + timestamp: 1_700_000_000_000, + messageId: "msg-history", + media: [{ path: historyPath, contentType: "image/png", kind: "image" }], + }, + { + sender: "@alice", + body: "fixed after refresh", + timestamp: 1_700_000_060_000, + messageId: "msg-after", + }, + ], + }, + }); + + const text = String(runTurnCall().text); + expect(text).toContain("describe current state"); + expect(text).toContain("Recent image 1 from @bob, message msg-history"); + expect(text).toContain("sent at 2023-11-14T22:13:20.000Z"); + expect(text).toContain("message 2 of 3 in available history"); + expect(text).not.toContain(historyPath); + expect(runTurnCall().attachments).toEqual([ + { + mediaType: "image/png", + data: historyImage.toString("base64"), + }, + ]); + }); + it("forwards media-understanding PDF page images alongside current image attachments", async () => { setReadyAcpResolution(); const currentPath = "/tmp/openclaw-current-image.png"; diff --git a/src/auto-reply/reply/history-media.ts b/src/auto-reply/reply/history-media.ts index e31aba1665fa..9cda7d4cc469 100644 --- a/src/auto-reply/reply/history-media.ts +++ b/src/auto-reply/reply/history-media.ts @@ -13,6 +13,9 @@ export type RecentInboundHistoryImage = { path: string; contentType: string; sender: string; + sentAtMs: number; + messagePosition: number; + messageCount: number; messageId?: string; }; @@ -101,6 +104,9 @@ export function resolveRecentInboundHistoryImages(params: { path: mediaPath, contentType, sender: entry.sender, + sentAtMs: timestamp, + messagePosition: index + 1, + messageCount: entries.length, ...(messageId ? { messageId } : {}), }); } @@ -108,6 +114,11 @@ export function resolveRecentInboundHistoryImages(params: { return out.toReversed(); } +function formatRecentHistoryImageSentAt(sentAtMs: number): string { + const date = new Date(sentAtMs); + return Number.isFinite(date.getTime()) ? date.toISOString() : `${sentAtMs}ms since epoch`; +} + export function appendRecentHistoryImageContext(params: { promptText: string; images: RecentInboundHistoryImage[]; @@ -117,7 +128,8 @@ export function appendRecentHistoryImageContext(params: { } const notes = params.images.map((image, index) => { const message = image.messageId ? `, message ${image.messageId}` : ""; - return `[Recent image ${index + 1} from ${image.sender}${message}, attached as media.]`; + const sentAt = formatRecentHistoryImageSentAt(image.sentAtMs); + return `[Recent image ${index + 1} from ${image.sender}${message}, sent at ${sentAt}, message ${image.messagePosition} of ${image.messageCount} in available history, attached as media.]`; }); return [params.promptText, notes.join("\n")] .filter((part) => part.trim().length > 0)