fix(whatsapp): restore interactive mentions and inbound media variants (#116901)

Co-authored-by: Peter Steinberger <steipete@macos.shared>
2026-08-24 19:35:28 -06:00 · 2026-07-31 08:41:00 -07:00
parent 8c16b18a79
commit e922b53675
6 changed files with 406 additions and 7 deletions
@@ -383,6 +383,144 @@ describe("web inbound media saves with extension", () => {
    await listener.close();
  });

+  it("preserves self-authored quoted media through the real Baileys reupload boundary", async () => {
+    const onMessage = vi.fn();
+    const listener = await monitorWebInbox({
+      cfg: { channels: { whatsapp: { allowFrom: ["*"] } } } as never,
+      verbose: false,
+      onMessage,
+      accountId: "default",
+      authDir: path.join(HOME, "wa-auth"),
+    });
+    const realSock = await getMockSocket();
+
+    realSock.ev.emit("messages.upsert", {
+      type: "notify",
+      messages: [
+        {
+          key: { id: "quote-own-image", fromMe: false, remoteJid: "111@s.whatsapp.net" },
+          message: {
+            extendedTextMessage: {
+              text: "what is in your image?",
+              contextInfo: {
+                stanzaId: "bot-image",
+                participant: "me@s.whatsapp.net",
+                quotedMessage: { imageMessage: { mimetype: "image/jpeg" } },
+              },
+            },
+          },
+          messageTimestamp: 1_700_000_007,
+        },
+      ],
+    });
+
+    await waitForMessage(onMessage);
+    const quoted = downloadMediaMessageMock.mock.calls[0]?.[0] as
+      | { key?: { fromMe?: boolean; id?: string; remoteJid?: string } }
+      | undefined;
+    expect(quoted?.key).toMatchObject({ fromMe: true, id: "bot-image" });
+
+    const { encryptMediaRetryRequest } = await vi.importActual<typeof import("baileys")>("baileys");
+    const retry = encryptMediaRetryRequest(
+      quoted!.key as never,
+      Buffer.alloc(32, 1),
+      "me@s.whatsapp.net",
+    );
+    const retryNode = Array.isArray(retry.content)
+      ? retry.content.find((node) => node.tag === "rmr")
+      : undefined;
+    expect(retryNode?.attrs.from_me).toBe("true");
+
+    await listener.close();
+  });
+
+  it("delivers incoming video notes as normal video media", async () => {
+    const onMessage = vi.fn();
+    const listener = await monitorWebInbox({
+      cfg: { channels: { whatsapp: { allowFrom: ["*"] } } } as never,
+      verbose: false,
+      onMessage,
+      accountId: "default",
+      authDir: path.join(HOME, "wa-auth"),
+    });
+    const realSock = await getMockSocket();
+
+    realSock.ev.emit("messages.upsert", {
+      type: "notify",
+      messages: [
+        {
+          key: { id: "video-note-1", fromMe: false, remoteJid: "111@s.whatsapp.net" },
+          message: { ptvMessage: { mimetype: "video/mp4" } },
+          messageTimestamp: 1_700_000_008,
+        },
+      ],
+    });
+
+    const inbound = await waitForMessage(onMessage);
+    expect(inbound.payload.media).toMatchObject({ kind: "video", type: "video/mp4" });
+    expect(inbound.payload.media?.path).toBeTruthy();
+    expect(downloadMediaMessageMock).toHaveBeenCalled();
+
+    await listener.close();
+  });
+
+  it("delivers native polls and preserves their questions when quoted", async () => {
+    const onMessage = vi.fn();
+    const listener = await monitorWebInbox({
+      cfg: { channels: { whatsapp: { allowFrom: ["*"] } } } as never,
+      verbose: false,
+      onMessage,
+      accountId: "default",
+      authDir: path.join(HOME, "wa-auth"),
+    });
+    const realSock = await getMockSocket();
+    const poll = {
+      name: "Lunch?",
+      options: [{ optionName: "Pizza" }, { optionName: "Sushi" }],
+    };
+
+    realSock.ev.emit("messages.upsert", {
+      type: "notify",
+      messages: [
+        {
+          key: { id: "poll-1", fromMe: false, remoteJid: "111@s.whatsapp.net" },
+          message: { pollCreationMessageV3: poll },
+          messageTimestamp: 1_700_000_009,
+        },
+      ],
+    });
+
+    expect((await waitForMessage(onMessage)).payload.body).toBe("Lunch?\n- Pizza\n- Sushi");
+    onMessage.mockClear();
+
+    realSock.ev.emit("messages.upsert", {
+      type: "notify",
+      messages: [
+        {
+          key: { id: "poll-reply", fromMe: false, remoteJid: "111@s.whatsapp.net" },
+          message: {
+            extendedTextMessage: {
+              text: "Pizza, please",
+              contextInfo: {
+                stanzaId: "poll-1",
+                participant: "111@s.whatsapp.net",
+                quotedMessage: { pollCreationMessageV3: poll },
+              },
+            },
+          },
+          messageTimestamp: 1_700_000_010,
+        },
+      ],
+    });
+
+    expect((await waitForMessage(onMessage)).quote).toMatchObject({
+      id: "poll-1",
+      body: "Lunch?\n- Pizza\n- Sushi",
+    });
+
+    await listener.close();
+  });
+
  it("passes mediaMaxMb to saveMediaStream", async () => {
    const onMessage = vi.fn();
    const listener = await monitorWebInbox({
@@ -1,8 +1,9 @@
 // Whatsapp tests cover extract plugin behavior.
-import type { proto } from "baileys";
+import { generateWAMessageFromContent, type proto } from "baileys";
 import { describe, expect, it } from "vitest";
 import {
  describeReplyContext,
+  extractMediaKind,
  extractMentionedJids,
  extractText,
  hasInboundUserContent,
@@ -77,6 +78,49 @@ describe("extractMentionedJids", () => {
    expect(extractMentionedJids(message)).toEqual([botJid]);
  });

+  it.each([
+    {
+      name: "template button replies",
+      message: {
+        templateButtonReplyMessage: {
+          selectedId: "confirm",
+          selectedDisplayText: "Confirm",
+          contextInfo: { mentionedJid: [botJid] },
+        },
+      },
+    },
+    {
+      name: "native interactive responses",
+      message: {
+        interactiveResponseMessage: {
+          body: { text: "Continue" },
+          contextInfo: { mentionedJid: [botJid] },
+        },
+      },
+    },
+    {
+      name: "video notes",
+      message: {
+        ptvMessage: {
+          mimetype: "video/mp4",
+          contextInfo: { mentionedJid: [botJid] },
+        },
+      },
+    },
+    {
+      name: "native polls",
+      message: {
+        pollCreationMessageV3: {
+          name: "Lunch?",
+          options: [{ optionName: "Pizza" }],
+          contextInfo: { mentionedJid: [botJid] },
+        },
+      },
+    },
+  ])("preserves direct bot mentions from $name", ({ message }) => {
+    expect(extractMentionedJids(message as proto.IMessage)).toEqual([botJid]);
+  });
+
  it("returns undefined for messages with no mentions", () => {
    const message: proto.IMessage = {
      extendedTextMessage: {
@@ -156,6 +200,37 @@ describe("describeReplyContext", () => {
      }),
    ).toBeNull();
  });
+
+  it("preserves a quoted poll encoded through the real Baileys message generator", () => {
+    const userJid = "15555550123@s.whatsapp.net";
+    const generated = generateWAMessageFromContent(
+      "120363000000000000@g.us",
+      { extendedTextMessage: { text: "Choose pizza" } },
+      {
+        userJid,
+        quoted: {
+          key: {
+            id: "original-poll",
+            remoteJid: "120363000000000000@g.us",
+            participant: userJid,
+            fromMe: false,
+          },
+          message: {
+            pollCreationMessageV3: {
+              name: "Lunch?",
+              options: [{ optionName: "Pizza" }, { optionName: "Sushi" }],
+            },
+          },
+        },
+      },
+    );
+
+    expect(describeReplyContext(generated.message ?? undefined)).toMatchObject({
+      id: "original-poll",
+      body: "Lunch?\n- Pizza\n- Sushi",
+      sender: { jid: userJid },
+    });
+  });
 });

 describe("extractText", () => {
@@ -248,6 +323,65 @@ describe("extractText", () => {
      },
      expected: "OK",
    },
+    {
+      name: "native multiple-choice poll",
+      message: {
+        pollCreationMessage: {
+          name: "Lunch?",
+          options: [{ optionName: "Pizza" }, { optionName: "Sushi" }],
+        },
+      },
+      expected: "Lunch?\n- Pizza\n- Sushi",
+    },
+    {
+      name: "native announcement-group poll",
+      message: {
+        pollCreationMessageV2: {
+          name: "Lunch?",
+          options: [{ optionName: "Pizza" }],
+        },
+      },
+      expected: "Lunch?\n- Pizza",
+    },
+    {
+      name: "native single-select poll",
+      message: {
+        pollCreationMessageV3: {
+          name: "Lunch?",
+          options: [{ optionName: "Pizza" }],
+        },
+      },
+      expected: "Lunch?\n- Pizza",
+    },
+    {
+      name: "future-proof native poll",
+      message: {
+        pollCreationMessageV4: {
+          message: {
+            pollCreationMessageV3: {
+              name: "Lunch?",
+              options: [{ optionName: "Pizza" }],
+            },
+          },
+        },
+      },
+      expected: "Lunch?\n- Pizza",
+    },
+    {
+      name: "native poll with blank options filtered",
+      message: {
+        pollCreationMessageV5: {
+          name: " Lunch? ",
+          options: [{ optionName: " " }, { optionName: " Pizza " }],
+        },
+      },
+      expected: "Lunch?\n- Pizza",
+    },
+    {
+      name: "video-note caption",
+      message: { ptvMessage: { caption: "Watch this", mimetype: "video/mp4" } },
+      expected: "Watch this",
+    },
  ])("preserves $name as inbound message text", ({ message, expected }) => {
    expect(extractText(message as proto.IMessage)).toBe(expected);
  });
@@ -296,6 +430,26 @@ describe("hasInboundUserContent", () => {
    ).toBe(true);
  });

+  it("classifies captionless video notes as user-visible video media", () => {
+    const message = { ptvMessage: { mimetype: "video/mp4" } } as proto.IMessage;
+
+    expect(extractMediaKind(message)).toBe("video");
+    expect(hasInboundUserContent(message)).toBe(true);
+  });
+
+  it.each([
+    "pollCreationMessage",
+    "pollCreationMessageV2",
+    "pollCreationMessageV3",
+    "pollCreationMessageV5",
+  ] as const)("admits populated %s as user-visible content", (pollKey) => {
+    expect(
+      hasInboundUserContent({
+        [pollKey]: { name: "Lunch?", options: [{ optionName: "Pizza" }] },
+      } as proto.IMessage),
+    ).toBe(true);
+  });
+
  it("returns true for audio message", () => {
    expect(
      hasInboundUserContent({ audioMessage: { mimetype: "audio/ogg" } } as proto.IMessage),
@@ -450,4 +604,12 @@ describe("hasInboundUserContent", () => {
      false,
    );
  });
+
+  it("does not admit an empty native poll envelope", () => {
+    expect(
+      hasInboundUserContent({
+        pollCreationMessage: { name: " ", options: [{ optionName: " " }] },
+      } as proto.IMessage),
+    ).toBe(false);
+  });
 });
@@ -122,11 +122,18 @@ export function extractMentionedJids(rawMessage: proto.IMessage | undefined): st
    message.extendedTextMessage?.contextInfo?.mentionedJid,
    message.imageMessage?.contextInfo?.mentionedJid,
    message.videoMessage?.contextInfo?.mentionedJid,
+    message.ptvMessage?.contextInfo?.mentionedJid,
    message.documentMessage?.contextInfo?.mentionedJid,
    message.audioMessage?.contextInfo?.mentionedJid,
    message.stickerMessage?.contextInfo?.mentionedJid,
    message.buttonsResponseMessage?.contextInfo?.mentionedJid,
    message.listResponseMessage?.contextInfo?.mentionedJid,
+    message.templateButtonReplyMessage?.contextInfo?.mentionedJid,
+    message.interactiveResponseMessage?.contextInfo?.mentionedJid,
+    message.pollCreationMessage?.contextInfo?.mentionedJid,
+    message.pollCreationMessageV2?.contextInfo?.mentionedJid,
+    message.pollCreationMessageV3?.contextInfo?.mentionedJid,
+    message.pollCreationMessageV5?.contextInfo?.mentionedJid,
  ];

  const flattened = candidates.flatMap((arr) => arr ?? []).filter(Boolean);
@@ -177,6 +184,7 @@ export function extractText(rawMessage: proto.IMessage | undefined): string | un
    const caption =
      candidate.imageMessage?.caption ??
      candidate.videoMessage?.caption ??
+      candidate.ptvMessage?.caption ??
      candidate.documentMessage?.caption;
    if (caption?.trim()) {
      return caption.trim();
@@ -194,6 +202,23 @@ export function extractText(rawMessage: proto.IMessage | undefined): string | un
    if (interactiveSelection) {
      return interactiveSelection.trim();
    }
+    const poll =
+      candidate.pollCreationMessage ??
+      candidate.pollCreationMessageV2 ??
+      candidate.pollCreationMessageV3 ??
+      candidate.pollCreationMessageV5;
+    if (poll) {
+      const question = poll.name?.trim();
+      const options = (poll.options ?? [])
+        .map((option) => option.optionName?.trim())
+        .filter((option): option is string => Boolean(option));
+      const pollText = [question, ...options.map((option) => `- ${option}`)]
+        .filter(Boolean)
+        .join("\n");
+      if (pollText) {
+        return pollText;
+      }
+    }
  }
  const contactPlaceholder =
    extractContactPlaceholder(message) ??
@@ -236,7 +261,7 @@ export function extractMediaKind(
  if (message.imageMessage) {
    return "image";
  }
-  if (message.videoMessage) {
+  if (message.videoMessage || message.ptvMessage) {
    // GIF playback is a video transport detail; no downstream behavior needs a new GIF kind.
    return "video";
  }
@@ -9,6 +9,7 @@ export function resolveInboundMediaMimetype(message: proto.IMessage): string | u
  const explicit =
    message.imageMessage?.mimetype ??
    message.videoMessage?.mimetype ??
+    message.ptvMessage?.mimetype ??
    message.documentMessage?.mimetype ??
    message.audioMessage?.mimetype ??
    message.stickerMessage?.mimetype ??
@@ -23,7 +24,7 @@ export function resolveInboundMediaMimetype(message: proto.IMessage): string | u
  if (message.imageMessage) {
    return "image/jpeg";
  }
-  if (message.videoMessage) {
+  if (message.videoMessage || message.ptvMessage) {
    return "video/mp4";
  }
  if (message.stickerMessage) {
@@ -1,11 +1,23 @@
 // Whatsapp tests cover media plugin behavior.
 import { Readable } from "node:stream";
 import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
-import { mockNormalizeMessageContent } from "../../../../test/mocks/baileys.js";
+import {
+  mockExtractMessageContent,
+  mockGetContentType,
+  mockNormalizeMessageContent,
+} from "../../../../test/mocks/baileys.js";

 type MockMessageInput = Parameters<typeof mockNormalizeMessageContent>[0];

-const { normalizeMessageContent, downloadMediaMessage, saveMediaStream } = vi.hoisted(() => ({
+const {
+  extractMessageContent,
+  getContentType,
+  normalizeMessageContent,
+  downloadMediaMessage,
+  saveMediaStream,
+} = vi.hoisted(() => ({
+  extractMessageContent: vi.fn((msg: MockMessageInput) => mockExtractMessageContent(msg)),
+  getContentType: vi.fn((msg: MockMessageInput) => mockGetContentType(msg)),
  normalizeMessageContent: vi.fn((msg: MockMessageInput) => mockNormalizeMessageContent(msg)),
  downloadMediaMessage: vi.fn().mockResolvedValue(Buffer.from("fake-media-data")),
  saveMediaStream: vi.fn(),
@@ -14,6 +26,8 @@ const { normalizeMessageContent, downloadMediaMessage, saveMediaStream } = vi.ho
 vi.mock("baileys", async () => {
  return {
    DisconnectReason: { loggedOut: 401 },
+    extractMessageContent,
+    getContentType,
    normalizeMessageContent,
    downloadMediaMessage,
  };
@@ -24,10 +38,16 @@ vi.mock("openclaw/plugin-sdk/media-store", () => ({
 }));

 let downloadInboundMedia: typeof import("./media.js").downloadInboundMedia;
+let downloadQuotedInboundMedia: typeof import("./media.js").downloadQuotedInboundMedia;

 const mockSock = {
  updateMediaMessage: vi.fn(),
  logger: { child: () => ({}) },
+  user: {
+    id: "15559876543:7@s.whatsapp.net",
+    lid: "277038292303944@lid",
+    phoneNumber: "15559876543@s.whatsapp.net",
+  },
 };

 async function expectMimetype(message: Record<string, unknown>, expected: string) {
@@ -46,7 +66,7 @@ async function expectMimetype(message: Record<string, unknown>, expected: string

 describe("downloadInboundMedia", () => {
  beforeAll(async () => {
-    ({ downloadInboundMedia } = await import("./media.js"));
+    ({ downloadInboundMedia, downloadQuotedInboundMedia } = await import("./media.js"));
  });

  beforeEach(() => {
@@ -98,6 +118,7 @@ describe("downloadInboundMedia", () => {
  it.each([
    { name: "image", message: { imageMessage: {} }, mimetype: "image/jpeg" },
    { name: "video", message: { videoMessage: {} }, mimetype: "video/mp4" },
+    { name: "video note", message: { ptvMessage: {} }, mimetype: "video/mp4" },
    { name: "sticker", message: { stickerMessage: {} }, mimetype: "image/webp" },
  ])("defaults MIME for $name messages without explicit MIME", async ({ message, mimetype }) => {
    await expectMimetype(message, mimetype);
@@ -147,4 +168,48 @@ describe("downloadInboundMedia", () => {
      ),
    ).rejects.toThrow("expired media reference");
  });
+
+  it.each([
+    {
+      name: "the bot's device-scoped phone identity",
+      participant: "15559876543@s.whatsapp.net",
+      expectedFromMe: true,
+    },
+    {
+      name: "the bot's LID identity",
+      participant: "277038292303944@lid",
+      expectedFromMe: true,
+    },
+    {
+      name: "another sender",
+      participant: "15550000000@s.whatsapp.net",
+      expectedFromMe: false,
+    },
+  ])("preserves quoted media authorship for $name", async ({ participant, expectedFromMe }) => {
+    await downloadQuotedInboundMedia(
+      {
+        key: { remoteJid: "120363000000000000@g.us", id: "reply-1", fromMe: false },
+        message: {
+          extendedTextMessage: {
+            text: "inspect this",
+            contextInfo: {
+              stanzaId: "quoted-image",
+              participant,
+              quotedMessage: { imageMessage: { mimetype: "image/jpeg" } },
+            },
+          },
+        },
+      } as never,
+      mockSock as never,
+    );
+
+    const quotedMessage = downloadMediaMessage.mock.calls[0]?.[0] as
+      | { key?: { fromMe?: boolean; id?: string; participant?: string } }
+      | undefined;
+    expect(quotedMessage?.key).toMatchObject({
+      fromMe: expectedFromMe,
+      id: "quoted-image",
+      participant,
+    });
+  });
 });
@@ -1,6 +1,7 @@
 // Whatsapp plugin module implements media behavior.
 import type { proto, WAMessage } from "baileys";
 import { saveMediaStream, type SavedMedia } from "openclaw/plugin-sdk/media-store";
+import { identitiesOverlap } from "../identity.js";
 import type { createWaSocket } from "../session.js";
 import { extractContextInfo } from "./extract.js";
 import { resolveInboundMediaMimetype } from "./media-mimetype.js";
@@ -32,6 +33,7 @@ export async function downloadInboundMedia(
  if (
    !message.imageMessage &&
    !message.videoMessage &&
+    !message.ptvMessage &&
    !message.documentMessage &&
    !message.audioMessage &&
    !message.stickerMessage
@@ -73,13 +75,19 @@ export async function downloadQuotedInboundMedia(
    return undefined;
  }
  const quotedMessage = contextInfo.quotedMessage;
+  const self = sock.user;
+  // Baileys copies fromMe into the media-reupload receipt; own quoted media must retain its author.
+  const quotedFromMe = identitiesOverlap(
+    { jid: contextInfo.participant },
+    { jid: self?.id, lid: self?.lid, e164: self?.phoneNumber },
+  );
  return downloadInboundMedia(
    {
      key: {
        id: contextInfo?.stanzaId || undefined,
        remoteJid: contextInfo.remoteJid ?? msg.key?.remoteJid ?? undefined,
        participant: contextInfo?.participant ?? undefined,
-        fromMe: false,
+        fromMe: quotedFromMe,
      },
      message: quotedMessage,
      messageTimestamp: msg.messageTimestamp,