diff --git a/extensions/whatsapp/src/inbound.media.test.ts b/extensions/whatsapp/src/inbound.media.test.ts index 4ce81e8d7899..5e0e94744775 100644 --- a/extensions/whatsapp/src/inbound.media.test.ts +++ b/extensions/whatsapp/src/inbound.media.test.ts @@ -383,6 +383,144 @@ describe("web inbound media saves with extension", () => { await listener.close(); }); + it("preserves self-authored quoted media through the real Baileys reupload boundary", async () => { + const onMessage = vi.fn(); + const listener = await monitorWebInbox({ + cfg: { channels: { whatsapp: { allowFrom: ["*"] } } } as never, + verbose: false, + onMessage, + accountId: "default", + authDir: path.join(HOME, "wa-auth"), + }); + const realSock = await getMockSocket(); + + realSock.ev.emit("messages.upsert", { + type: "notify", + messages: [ + { + key: { id: "quote-own-image", fromMe: false, remoteJid: "111@s.whatsapp.net" }, + message: { + extendedTextMessage: { + text: "what is in your image?", + contextInfo: { + stanzaId: "bot-image", + participant: "me@s.whatsapp.net", + quotedMessage: { imageMessage: { mimetype: "image/jpeg" } }, + }, + }, + }, + messageTimestamp: 1_700_000_007, + }, + ], + }); + + await waitForMessage(onMessage); + const quoted = downloadMediaMessageMock.mock.calls[0]?.[0] as + | { key?: { fromMe?: boolean; id?: string; remoteJid?: string } } + | undefined; + expect(quoted?.key).toMatchObject({ fromMe: true, id: "bot-image" }); + + const { encryptMediaRetryRequest } = await vi.importActual("baileys"); + const retry = encryptMediaRetryRequest( + quoted!.key as never, + Buffer.alloc(32, 1), + "me@s.whatsapp.net", + ); + const retryNode = Array.isArray(retry.content) + ? retry.content.find((node) => node.tag === "rmr") + : undefined; + expect(retryNode?.attrs.from_me).toBe("true"); + + await listener.close(); + }); + + it("delivers incoming video notes as normal video media", async () => { + const onMessage = vi.fn(); + const listener = await monitorWebInbox({ + cfg: { channels: { whatsapp: { allowFrom: ["*"] } } } as never, + verbose: false, + onMessage, + accountId: "default", + authDir: path.join(HOME, "wa-auth"), + }); + const realSock = await getMockSocket(); + + realSock.ev.emit("messages.upsert", { + type: "notify", + messages: [ + { + key: { id: "video-note-1", fromMe: false, remoteJid: "111@s.whatsapp.net" }, + message: { ptvMessage: { mimetype: "video/mp4" } }, + messageTimestamp: 1_700_000_008, + }, + ], + }); + + const inbound = await waitForMessage(onMessage); + expect(inbound.payload.media).toMatchObject({ kind: "video", type: "video/mp4" }); + expect(inbound.payload.media?.path).toBeTruthy(); + expect(downloadMediaMessageMock).toHaveBeenCalled(); + + await listener.close(); + }); + + it("delivers native polls and preserves their questions when quoted", async () => { + const onMessage = vi.fn(); + const listener = await monitorWebInbox({ + cfg: { channels: { whatsapp: { allowFrom: ["*"] } } } as never, + verbose: false, + onMessage, + accountId: "default", + authDir: path.join(HOME, "wa-auth"), + }); + const realSock = await getMockSocket(); + const poll = { + name: "Lunch?", + options: [{ optionName: "Pizza" }, { optionName: "Sushi" }], + }; + + realSock.ev.emit("messages.upsert", { + type: "notify", + messages: [ + { + key: { id: "poll-1", fromMe: false, remoteJid: "111@s.whatsapp.net" }, + message: { pollCreationMessageV3: poll }, + messageTimestamp: 1_700_000_009, + }, + ], + }); + + expect((await waitForMessage(onMessage)).payload.body).toBe("Lunch?\n- Pizza\n- Sushi"); + onMessage.mockClear(); + + realSock.ev.emit("messages.upsert", { + type: "notify", + messages: [ + { + key: { id: "poll-reply", fromMe: false, remoteJid: "111@s.whatsapp.net" }, + message: { + extendedTextMessage: { + text: "Pizza, please", + contextInfo: { + stanzaId: "poll-1", + participant: "111@s.whatsapp.net", + quotedMessage: { pollCreationMessageV3: poll }, + }, + }, + }, + messageTimestamp: 1_700_000_010, + }, + ], + }); + + expect((await waitForMessage(onMessage)).quote).toMatchObject({ + id: "poll-1", + body: "Lunch?\n- Pizza\n- Sushi", + }); + + await listener.close(); + }); + it("passes mediaMaxMb to saveMediaStream", async () => { const onMessage = vi.fn(); const listener = await monitorWebInbox({ diff --git a/extensions/whatsapp/src/inbound/extract.test.ts b/extensions/whatsapp/src/inbound/extract.test.ts index 7e5a7b8a6566..45bec4ce7c9c 100644 --- a/extensions/whatsapp/src/inbound/extract.test.ts +++ b/extensions/whatsapp/src/inbound/extract.test.ts @@ -1,8 +1,9 @@ // Whatsapp tests cover extract plugin behavior. -import type { proto } from "baileys"; +import { generateWAMessageFromContent, type proto } from "baileys"; import { describe, expect, it } from "vitest"; import { describeReplyContext, + extractMediaKind, extractMentionedJids, extractText, hasInboundUserContent, @@ -77,6 +78,49 @@ describe("extractMentionedJids", () => { expect(extractMentionedJids(message)).toEqual([botJid]); }); + it.each([ + { + name: "template button replies", + message: { + templateButtonReplyMessage: { + selectedId: "confirm", + selectedDisplayText: "Confirm", + contextInfo: { mentionedJid: [botJid] }, + }, + }, + }, + { + name: "native interactive responses", + message: { + interactiveResponseMessage: { + body: { text: "Continue" }, + contextInfo: { mentionedJid: [botJid] }, + }, + }, + }, + { + name: "video notes", + message: { + ptvMessage: { + mimetype: "video/mp4", + contextInfo: { mentionedJid: [botJid] }, + }, + }, + }, + { + name: "native polls", + message: { + pollCreationMessageV3: { + name: "Lunch?", + options: [{ optionName: "Pizza" }], + contextInfo: { mentionedJid: [botJid] }, + }, + }, + }, + ])("preserves direct bot mentions from $name", ({ message }) => { + expect(extractMentionedJids(message as proto.IMessage)).toEqual([botJid]); + }); + it("returns undefined for messages with no mentions", () => { const message: proto.IMessage = { extendedTextMessage: { @@ -156,6 +200,37 @@ describe("describeReplyContext", () => { }), ).toBeNull(); }); + + it("preserves a quoted poll encoded through the real Baileys message generator", () => { + const userJid = "15555550123@s.whatsapp.net"; + const generated = generateWAMessageFromContent( + "120363000000000000@g.us", + { extendedTextMessage: { text: "Choose pizza" } }, + { + userJid, + quoted: { + key: { + id: "original-poll", + remoteJid: "120363000000000000@g.us", + participant: userJid, + fromMe: false, + }, + message: { + pollCreationMessageV3: { + name: "Lunch?", + options: [{ optionName: "Pizza" }, { optionName: "Sushi" }], + }, + }, + }, + }, + ); + + expect(describeReplyContext(generated.message ?? undefined)).toMatchObject({ + id: "original-poll", + body: "Lunch?\n- Pizza\n- Sushi", + sender: { jid: userJid }, + }); + }); }); describe("extractText", () => { @@ -248,6 +323,65 @@ describe("extractText", () => { }, expected: "OK", }, + { + name: "native multiple-choice poll", + message: { + pollCreationMessage: { + name: "Lunch?", + options: [{ optionName: "Pizza" }, { optionName: "Sushi" }], + }, + }, + expected: "Lunch?\n- Pizza\n- Sushi", + }, + { + name: "native announcement-group poll", + message: { + pollCreationMessageV2: { + name: "Lunch?", + options: [{ optionName: "Pizza" }], + }, + }, + expected: "Lunch?\n- Pizza", + }, + { + name: "native single-select poll", + message: { + pollCreationMessageV3: { + name: "Lunch?", + options: [{ optionName: "Pizza" }], + }, + }, + expected: "Lunch?\n- Pizza", + }, + { + name: "future-proof native poll", + message: { + pollCreationMessageV4: { + message: { + pollCreationMessageV3: { + name: "Lunch?", + options: [{ optionName: "Pizza" }], + }, + }, + }, + }, + expected: "Lunch?\n- Pizza", + }, + { + name: "native poll with blank options filtered", + message: { + pollCreationMessageV5: { + name: " Lunch? ", + options: [{ optionName: " " }, { optionName: " Pizza " }], + }, + }, + expected: "Lunch?\n- Pizza", + }, + { + name: "video-note caption", + message: { ptvMessage: { caption: "Watch this", mimetype: "video/mp4" } }, + expected: "Watch this", + }, ])("preserves $name as inbound message text", ({ message, expected }) => { expect(extractText(message as proto.IMessage)).toBe(expected); }); @@ -296,6 +430,26 @@ describe("hasInboundUserContent", () => { ).toBe(true); }); + it("classifies captionless video notes as user-visible video media", () => { + const message = { ptvMessage: { mimetype: "video/mp4" } } as proto.IMessage; + + expect(extractMediaKind(message)).toBe("video"); + expect(hasInboundUserContent(message)).toBe(true); + }); + + it.each([ + "pollCreationMessage", + "pollCreationMessageV2", + "pollCreationMessageV3", + "pollCreationMessageV5", + ] as const)("admits populated %s as user-visible content", (pollKey) => { + expect( + hasInboundUserContent({ + [pollKey]: { name: "Lunch?", options: [{ optionName: "Pizza" }] }, + } as proto.IMessage), + ).toBe(true); + }); + it("returns true for audio message", () => { expect( hasInboundUserContent({ audioMessage: { mimetype: "audio/ogg" } } as proto.IMessage), @@ -450,4 +604,12 @@ describe("hasInboundUserContent", () => { false, ); }); + + it("does not admit an empty native poll envelope", () => { + expect( + hasInboundUserContent({ + pollCreationMessage: { name: " ", options: [{ optionName: " " }] }, + } as proto.IMessage), + ).toBe(false); + }); }); diff --git a/extensions/whatsapp/src/inbound/extract.ts b/extensions/whatsapp/src/inbound/extract.ts index 2b0a45d7c580..c9d55473d892 100644 --- a/extensions/whatsapp/src/inbound/extract.ts +++ b/extensions/whatsapp/src/inbound/extract.ts @@ -122,11 +122,18 @@ export function extractMentionedJids(rawMessage: proto.IMessage | undefined): st message.extendedTextMessage?.contextInfo?.mentionedJid, message.imageMessage?.contextInfo?.mentionedJid, message.videoMessage?.contextInfo?.mentionedJid, + message.ptvMessage?.contextInfo?.mentionedJid, message.documentMessage?.contextInfo?.mentionedJid, message.audioMessage?.contextInfo?.mentionedJid, message.stickerMessage?.contextInfo?.mentionedJid, message.buttonsResponseMessage?.contextInfo?.mentionedJid, message.listResponseMessage?.contextInfo?.mentionedJid, + message.templateButtonReplyMessage?.contextInfo?.mentionedJid, + message.interactiveResponseMessage?.contextInfo?.mentionedJid, + message.pollCreationMessage?.contextInfo?.mentionedJid, + message.pollCreationMessageV2?.contextInfo?.mentionedJid, + message.pollCreationMessageV3?.contextInfo?.mentionedJid, + message.pollCreationMessageV5?.contextInfo?.mentionedJid, ]; const flattened = candidates.flatMap((arr) => arr ?? []).filter(Boolean); @@ -177,6 +184,7 @@ export function extractText(rawMessage: proto.IMessage | undefined): string | un const caption = candidate.imageMessage?.caption ?? candidate.videoMessage?.caption ?? + candidate.ptvMessage?.caption ?? candidate.documentMessage?.caption; if (caption?.trim()) { return caption.trim(); @@ -194,6 +202,23 @@ export function extractText(rawMessage: proto.IMessage | undefined): string | un if (interactiveSelection) { return interactiveSelection.trim(); } + const poll = + candidate.pollCreationMessage ?? + candidate.pollCreationMessageV2 ?? + candidate.pollCreationMessageV3 ?? + candidate.pollCreationMessageV5; + if (poll) { + const question = poll.name?.trim(); + const options = (poll.options ?? []) + .map((option) => option.optionName?.trim()) + .filter((option): option is string => Boolean(option)); + const pollText = [question, ...options.map((option) => `- ${option}`)] + .filter(Boolean) + .join("\n"); + if (pollText) { + return pollText; + } + } } const contactPlaceholder = extractContactPlaceholder(message) ?? @@ -236,7 +261,7 @@ export function extractMediaKind( if (message.imageMessage) { return "image"; } - if (message.videoMessage) { + if (message.videoMessage || message.ptvMessage) { // GIF playback is a video transport detail; no downstream behavior needs a new GIF kind. return "video"; } diff --git a/extensions/whatsapp/src/inbound/media-mimetype.ts b/extensions/whatsapp/src/inbound/media-mimetype.ts index 607972e1fe3a..fe4098e41852 100644 --- a/extensions/whatsapp/src/inbound/media-mimetype.ts +++ b/extensions/whatsapp/src/inbound/media-mimetype.ts @@ -9,6 +9,7 @@ export function resolveInboundMediaMimetype(message: proto.IMessage): string | u const explicit = message.imageMessage?.mimetype ?? message.videoMessage?.mimetype ?? + message.ptvMessage?.mimetype ?? message.documentMessage?.mimetype ?? message.audioMessage?.mimetype ?? message.stickerMessage?.mimetype ?? @@ -23,7 +24,7 @@ export function resolveInboundMediaMimetype(message: proto.IMessage): string | u if (message.imageMessage) { return "image/jpeg"; } - if (message.videoMessage) { + if (message.videoMessage || message.ptvMessage) { return "video/mp4"; } if (message.stickerMessage) { diff --git a/extensions/whatsapp/src/inbound/media.node.test.ts b/extensions/whatsapp/src/inbound/media.node.test.ts index 1d1cd3a8e29e..365d735e71c5 100644 --- a/extensions/whatsapp/src/inbound/media.node.test.ts +++ b/extensions/whatsapp/src/inbound/media.node.test.ts @@ -1,11 +1,23 @@ // Whatsapp tests cover media plugin behavior. import { Readable } from "node:stream"; import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; -import { mockNormalizeMessageContent } from "../../../../test/mocks/baileys.js"; +import { + mockExtractMessageContent, + mockGetContentType, + mockNormalizeMessageContent, +} from "../../../../test/mocks/baileys.js"; type MockMessageInput = Parameters[0]; -const { normalizeMessageContent, downloadMediaMessage, saveMediaStream } = vi.hoisted(() => ({ +const { + extractMessageContent, + getContentType, + normalizeMessageContent, + downloadMediaMessage, + saveMediaStream, +} = vi.hoisted(() => ({ + extractMessageContent: vi.fn((msg: MockMessageInput) => mockExtractMessageContent(msg)), + getContentType: vi.fn((msg: MockMessageInput) => mockGetContentType(msg)), normalizeMessageContent: vi.fn((msg: MockMessageInput) => mockNormalizeMessageContent(msg)), downloadMediaMessage: vi.fn().mockResolvedValue(Buffer.from("fake-media-data")), saveMediaStream: vi.fn(), @@ -14,6 +26,8 @@ const { normalizeMessageContent, downloadMediaMessage, saveMediaStream } = vi.ho vi.mock("baileys", async () => { return { DisconnectReason: { loggedOut: 401 }, + extractMessageContent, + getContentType, normalizeMessageContent, downloadMediaMessage, }; @@ -24,10 +38,16 @@ vi.mock("openclaw/plugin-sdk/media-store", () => ({ })); let downloadInboundMedia: typeof import("./media.js").downloadInboundMedia; +let downloadQuotedInboundMedia: typeof import("./media.js").downloadQuotedInboundMedia; const mockSock = { updateMediaMessage: vi.fn(), logger: { child: () => ({}) }, + user: { + id: "15559876543:7@s.whatsapp.net", + lid: "277038292303944@lid", + phoneNumber: "15559876543@s.whatsapp.net", + }, }; async function expectMimetype(message: Record, expected: string) { @@ -46,7 +66,7 @@ async function expectMimetype(message: Record, expected: string describe("downloadInboundMedia", () => { beforeAll(async () => { - ({ downloadInboundMedia } = await import("./media.js")); + ({ downloadInboundMedia, downloadQuotedInboundMedia } = await import("./media.js")); }); beforeEach(() => { @@ -98,6 +118,7 @@ describe("downloadInboundMedia", () => { it.each([ { name: "image", message: { imageMessage: {} }, mimetype: "image/jpeg" }, { name: "video", message: { videoMessage: {} }, mimetype: "video/mp4" }, + { name: "video note", message: { ptvMessage: {} }, mimetype: "video/mp4" }, { name: "sticker", message: { stickerMessage: {} }, mimetype: "image/webp" }, ])("defaults MIME for $name messages without explicit MIME", async ({ message, mimetype }) => { await expectMimetype(message, mimetype); @@ -147,4 +168,48 @@ describe("downloadInboundMedia", () => { ), ).rejects.toThrow("expired media reference"); }); + + it.each([ + { + name: "the bot's device-scoped phone identity", + participant: "15559876543@s.whatsapp.net", + expectedFromMe: true, + }, + { + name: "the bot's LID identity", + participant: "277038292303944@lid", + expectedFromMe: true, + }, + { + name: "another sender", + participant: "15550000000@s.whatsapp.net", + expectedFromMe: false, + }, + ])("preserves quoted media authorship for $name", async ({ participant, expectedFromMe }) => { + await downloadQuotedInboundMedia( + { + key: { remoteJid: "120363000000000000@g.us", id: "reply-1", fromMe: false }, + message: { + extendedTextMessage: { + text: "inspect this", + contextInfo: { + stanzaId: "quoted-image", + participant, + quotedMessage: { imageMessage: { mimetype: "image/jpeg" } }, + }, + }, + }, + } as never, + mockSock as never, + ); + + const quotedMessage = downloadMediaMessage.mock.calls[0]?.[0] as + | { key?: { fromMe?: boolean; id?: string; participant?: string } } + | undefined; + expect(quotedMessage?.key).toMatchObject({ + fromMe: expectedFromMe, + id: "quoted-image", + participant, + }); + }); }); diff --git a/extensions/whatsapp/src/inbound/media.ts b/extensions/whatsapp/src/inbound/media.ts index 6f623e5bccca..05b07fef99d7 100644 --- a/extensions/whatsapp/src/inbound/media.ts +++ b/extensions/whatsapp/src/inbound/media.ts @@ -1,6 +1,7 @@ // Whatsapp plugin module implements media behavior. import type { proto, WAMessage } from "baileys"; import { saveMediaStream, type SavedMedia } from "openclaw/plugin-sdk/media-store"; +import { identitiesOverlap } from "../identity.js"; import type { createWaSocket } from "../session.js"; import { extractContextInfo } from "./extract.js"; import { resolveInboundMediaMimetype } from "./media-mimetype.js"; @@ -32,6 +33,7 @@ export async function downloadInboundMedia( if ( !message.imageMessage && !message.videoMessage && + !message.ptvMessage && !message.documentMessage && !message.audioMessage && !message.stickerMessage @@ -73,13 +75,19 @@ export async function downloadQuotedInboundMedia( return undefined; } const quotedMessage = contextInfo.quotedMessage; + const self = sock.user; + // Baileys copies fromMe into the media-reupload receipt; own quoted media must retain its author. + const quotedFromMe = identitiesOverlap( + { jid: contextInfo.participant }, + { jid: self?.id, lid: self?.lid, e164: self?.phoneNumber }, + ); return downloadInboundMedia( { key: { id: contextInfo?.stanzaId || undefined, remoteJid: contextInfo.remoteJid ?? msg.key?.remoteJid ?? undefined, participant: contextInfo?.participant ?? undefined, - fromMe: false, + fromMe: quotedFromMe, }, message: quotedMessage, messageTimestamp: msg.messageTimestamp,