feat(reply): annotate recent history images (#100866)

* feat(reply): annotate recent history images

* fix(reply): describe bounded history image positions

---------

Co-authored-by: martin <huangtongxuedev@outlook.com>
Co-authored-by: Vincent Koc <vincentkoc@ieee.org>

src/auto-reply/reply/dispatch-acp.test.ts

@@ -1216,24 +1216,36 @@ describe("tryDispatchAcpReply", () => {
         path: "/tmp/recent-2.png",
         contentType: "image/png",
         sender: "Recent 2",
+        sentAtMs: 1_699_999_997_000,
+        messagePosition: 6,
+        messageCount: 9,
         messageId: "recent-2",
       },
       {
         path: "/tmp/recent-3.png",
         contentType: "image/png",
         sender: "Recent 3",
+        sentAtMs: 1_699_999_998_000,
+        messagePosition: 7,
+        messageCount: 9,
         messageId: "recent-3",
       },
       {
         path: "/tmp/recent-4.png",
         contentType: "image/png",
         sender: "Recent 4",
+        sentAtMs: 1_699_999_999_000,
+        messagePosition: 8,
+        messageCount: 9,
         messageId: "recent-4",
       },
       {
         path: "C:\\Users\\Alice\\Pictures\\recent.png",
         contentType: "image/png",
         sender: "Windows",
+        sentAtMs: 1_699_999_999_500,
+        messagePosition: 9,
+        messageCount: 9,
         messageId: "windows",
       },
     ]);
@@ -1247,6 +1259,9 @@ describe("tryDispatchAcpReply", () => {
           path: "/tmp/secret.png",
           contentType: "image/png",
           sender: "@alice",
+          sentAtMs: 1_700_000_000_000,
+          messagePosition: 2,
+          messageCount: 5,
           messageId: "msg-1",
         },
       ],
@@ -1254,6 +1269,8 @@ describe("tryDispatchAcpReply", () => {
 
     expect(text).toContain("what is this?");
     expect(text).toContain("Recent image 1 from @alice, message msg-1");
+    expect(text).toContain("sent at 2023-11-14T22:13:20.000Z");
+    expect(text).toContain("message 2 of 5 in available history");
     expect(text).not.toContain("/tmp/secret.png");
   });
 
@@ -1309,6 +1326,9 @@ describe("tryDispatchAcpReply", () => {
           path: imagePath,
           contentType: "image/png",
           sender: "@alice",
+          sentAtMs: 1_700_000_000_000,
+          messagePosition: 1,
+          messageCount: 1,
           messageId: "msg-1",
         },
       ]);
@@ -1557,6 +1577,9 @@ describe("tryDispatchAcpReply", () => {
           path: historyPath,
           contentType: "image/png",
           sender: "@alice",
+          sentAtMs: 1_700_000_000_000,
+          messagePosition: 1,
+          messageCount: 1,
           messageId: "msg-history",
         },
       ]);
@@ -1593,6 +1616,54 @@ describe("tryDispatchAcpReply", () => {
     ]);
   });
 
+  it("annotates recent history images with sent time and available history position", async () => {
+    setReadyAcpResolution();
+    const historyPath = "/tmp/openclaw-history-metadata.png";
+    const historyImage = Buffer.from("history-image");
+    acpAttachmentBuffers.set(historyPath, historyImage);
+
+    await runDispatch({
+      bodyForAgent: "describe current state",
+      ctxOverrides: {
+        Timestamp: 1_700_000_060_000,
+        InboundHistory: [
+          {
+            sender: "@alice",
+            body: "bug report",
+            timestamp: 1_699_999_980_000,
+            messageId: "msg-before",
+          },
+          {
+            sender: "@bob",
+            body: "<media:image>",
+            timestamp: 1_700_000_000_000,
+            messageId: "msg-history",
+            media: [{ path: historyPath, contentType: "image/png", kind: "image" }],
+          },
+          {
+            sender: "@alice",
+            body: "fixed after refresh",
+            timestamp: 1_700_000_060_000,
+            messageId: "msg-after",
+          },
+        ],
+      },
+    });
+
+    const text = String(runTurnCall().text);
+    expect(text).toContain("describe current state");
+    expect(text).toContain("Recent image 1 from @bob, message msg-history");
+    expect(text).toContain("sent at 2023-11-14T22:13:20.000Z");
+    expect(text).toContain("message 2 of 3 in available history");
+    expect(text).not.toContain(historyPath);
+    expect(runTurnCall().attachments).toEqual([
+      {
+        mediaType: "image/png",
+        data: historyImage.toString("base64"),
+      },
+    ]);
+  });
+
   it("forwards media-understanding PDF page images alongside current image attachments", async () => {
     setReadyAcpResolution();
     const currentPath = "/tmp/openclaw-current-image.png";

src/auto-reply/reply/history-media.ts

@@ -13,6 +13,9 @@ export type RecentInboundHistoryImage = {
   path: string;
   contentType: string;
   sender: string;
+  sentAtMs: number;
+  messagePosition: number;
+  messageCount: number;
   messageId?: string;
 };
 
@@ -101,6 +104,9 @@ export function resolveRecentInboundHistoryImages(params: {
         path: mediaPath,
         contentType,
         sender: entry.sender,
+        sentAtMs: timestamp,
+        messagePosition: index + 1,
+        messageCount: entries.length,
         ...(messageId ? { messageId } : {}),
       });
     }
@@ -108,6 +114,11 @@ export function resolveRecentInboundHistoryImages(params: {
   return out.toReversed();
 }
 
+function formatRecentHistoryImageSentAt(sentAtMs: number): string {
+  const date = new Date(sentAtMs);
+  return Number.isFinite(date.getTime()) ? date.toISOString() : `${sentAtMs}ms since epoch`;
+}
+
 export function appendRecentHistoryImageContext(params: {
   promptText: string;
   images: RecentInboundHistoryImage[];
@@ -117,7 +128,8 @@ export function appendRecentHistoryImageContext(params: {
   }
   const notes = params.images.map((image, index) => {
     const message = image.messageId ? `, message ${image.messageId}` : "";
-    return `[Recent image ${index + 1} from ${image.sender}${message}, attached as media.]`;
+    const sentAt = formatRecentHistoryImageSentAt(image.sentAtMs);
+    return `[Recent image ${index + 1} from ${image.sender}${message}, sent at ${sentAt}, message ${image.messagePosition} of ${image.messageCount} in available history, attached as media.]`;
   });
   return [params.promptText, notes.join("\n")]
     .filter((part) => part.trim().length > 0)