diff --git a/docs/concepts/qa-e2e-automation.md b/docs/concepts/qa-e2e-automation.md index c203838194c0..90960941db3f 100644 --- a/docs/concepts/qa-e2e-automation.md +++ b/docs/concepts/qa-e2e-automation.md @@ -48,6 +48,7 @@ script aliases; both forms are supported. | `qa telegram` | Live transport lane against a real private Telegram group. | | `qa discord` | Live transport lane against a real private Discord guild channel. | | `qa slack` | Live transport lane against a real private Slack channel. | +| `qa whatsapp` | Live transport lane against real WhatsApp Web accounts. | | `qa mantis` | Before and after verification runner for live transport bugs, with Discord status-reactions evidence, Crabbox desktop/browser smoke, and Slack-in-VNC smoke. See [Mantis](/concepts/mantis) and [Mantis Slack Desktop Runbook](/concepts/mantis-slack-desktop-runbook). | ## Operator flow @@ -168,15 +169,16 @@ decision still comes from the Discord REST oracle. CI uses the same command surface in `.github/workflows/qa-live-transports-convex.yml`. Scheduled and default manual runs execute the fast Matrix profile with live frontier credentials, `--fast`, and `OPENCLAW_QA_MATRIX_NO_REPLY_WINDOW_MS=3000`. Manual `matrix_profile=all` fans out into the five profile shards so the exhaustive catalog can run in parallel while keeping one artifact directory per shard. -For transport-real Telegram, Discord, and Slack smoke lanes: +For transport-real Telegram, Discord, Slack, and WhatsApp smoke lanes: ```bash pnpm openclaw qa telegram pnpm openclaw qa discord pnpm openclaw qa slack +pnpm openclaw qa whatsapp ``` -They target a pre-existing real channel with two bots (driver + SUT). Required env vars, scenario lists, output artifacts, and the Convex credential pool are documented in [Telegram, Discord, and Slack QA reference](#telegram-discord-and-slack-qa-reference) below. +They target a pre-existing real channel with two bots or accounts (driver + SUT). Required env vars, scenario lists, output artifacts, and the Convex credential pool are documented in [Telegram, Discord, Slack, and WhatsApp QA reference](#telegram-discord-slack-and-whatsapp-qa-reference) below. For a full Slack desktop VM run with VNC rescue, run: @@ -276,10 +278,10 @@ coverage helpers, and scenario-selection helper from | Telegram | x | x | x | | | | | | | x | | | Discord | x | x | x | | | | | | | | x | | Slack | x | x | x | x | x | x | x | x | | | | +| WhatsApp | x | x | | x | x | x | | | x | x | | This keeps `qa-channel` as the broad product-behavior suite while Matrix, -Telegram, and future live transports share one explicit transport-contract -checklist. +Telegram, and other live transports share one explicit transport-contract checklist. For a disposable Linux VM lane without bringing Docker into the QA path, run: @@ -308,25 +310,25 @@ guest: env-based provider keys, the QA live provider config path, and `CODEX_HOME` when present. Keep `--output-dir` under the repo root so the guest can write back through the mounted workspace. -## Telegram, Discord, and Slack QA reference +## Telegram, Discord, Slack, and WhatsApp QA reference -Matrix has a [dedicated page](/concepts/qa-matrix) because of its scenario count and Docker-backed homeserver provisioning. Telegram, Discord, and Slack are smaller - a handful of scenarios each, no profile system, against pre-existing real channels - so their reference lives here. +Matrix has a [dedicated page](/concepts/qa-matrix) because of its scenario count and Docker-backed homeserver provisioning. Telegram, Discord, Slack, and WhatsApp run against pre-existing real transports, so their reference lives here. ### Shared CLI flags These lanes register through `extensions/qa-lab/src/live-transports/shared/live-transport-cli.ts` and accept the same flags: -| Flag | Default | Description | -| ------------------------------------- | --------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | -| `--scenario ` | - | Run only this scenario. Repeatable. | -| `--output-dir ` | `/.artifacts/qa-e2e/{telegram,discord,slack}-` | Where reports/summary/observed messages and the output log are written. Relative paths resolve against `--repo-root`. | -| `--repo-root ` | `process.cwd()` | Repository root when invoking from a neutral cwd. | -| `--sut-account ` | `sut` | Temporary account id inside the QA gateway config. | -| `--provider-mode ` | `live-frontier` | `mock-openai` or `live-frontier` (legacy `live-openai` still works). | -| `--model ` / `--alt-model ` | provider default | Primary/alternate model refs. | -| `--fast` | off | Provider fast mode where supported. | -| `--credential-source ` | `env` | See [Convex credential pool](#convex-credential-pool). | -| `--credential-role ` | `ci` in CI, `maintainer` otherwise | Role used when `--credential-source convex`. | +| Flag | Default | Description | +| ------------------------------------- | -------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | +| `--scenario ` | - | Run only this scenario. Repeatable. | +| `--output-dir ` | `/.artifacts/qa-e2e/-` | Where reports/summary/observed messages and the output log are written. Relative paths resolve against `--repo-root`. | +| `--repo-root ` | `process.cwd()` | Repository root when invoking from a neutral cwd. | +| `--sut-account ` | `sut` | Temporary account id inside the QA gateway config. | +| `--provider-mode ` | `live-frontier` | `mock-openai` or `live-frontier` (legacy `live-openai` still works). | +| `--model ` / `--alt-model ` | provider default | Primary/alternate model refs. | +| `--fast` | off | Provider fast mode where supported. | +| `--credential-source ` | `env` | See [Convex credential pool](#convex-credential-pool). | +| `--credential-role ` | `ci` in CI, `maintainer` otherwise | Role used when `--credential-source convex`. | Each lane exits non-zero on any failed scenario. `--allow-failures` writes artifacts without setting a failing exit code. @@ -688,22 +690,52 @@ Required env when `--credential-source env`: Optional: -- `OPENCLAW_QA_WHATSAPP_GROUP_JID` enables `whatsapp-mention-gating`. +- `OPENCLAW_QA_WHATSAPP_GROUP_JID` enables group scenarios such as + `whatsapp-mention-gating` and `whatsapp-group-allowlist-block`. - `OPENCLAW_QA_WHATSAPP_CAPTURE_CONTENT=1` keeps message bodies in observed-message artifacts. -Scenarios (`extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.ts`): +Scenario catalog (`extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.ts`): -- `whatsapp-canary` -- `whatsapp-pairing-block` -- `whatsapp-mention-gating` -- `whatsapp-approval-exec-native` - opt-in native WhatsApp exec approval - scenario. Requests an exec approval through the gateway, verifies the - WhatsApp message has native reaction approval affordances, resolves it, and - verifies the resolved WhatsApp follow-up. -- `whatsapp-approval-plugin-native` - opt-in native WhatsApp plugin approval - scenario. Enables exec and plugin approval forwarding together, then verifies - the same pending/resolved native WhatsApp path. +- Baseline and group gating: `whatsapp-canary`, `whatsapp-pairing-block`, + `whatsapp-mention-gating`, `whatsapp-top-level-reply-shape`, + `whatsapp-restart-resume`, `whatsapp-group-allowlist-block`. +- Native commands: `whatsapp-help-command`, `whatsapp-status-command`, + `whatsapp-commands-command`, `whatsapp-tools-compact-command`, + `whatsapp-whoami-command`, `whatsapp-context-command`, + `whatsapp-native-new-command`. +- Reply and final-output behavior: `whatsapp-tool-only-usage-footer`, + `whatsapp-reply-to-message`, `whatsapp-reply-context-isolation`, + `whatsapp-reply-delivery-shape`, `whatsapp-stream-final-message-accounting`. +- Inbound media and structured messages: `whatsapp-inbound-image-caption`, + `whatsapp-audio-preflight`, `whatsapp-inbound-structured-messages`, + `whatsapp-group-audio-gating`. These send real WhatsApp image, audio, + document, location, contact, and sticker events through the driver. +- Outbound Gateway and message action coverage: + `whatsapp-outbound-media-matrix`, + `whatsapp-outbound-document-preserves-filename`, `whatsapp-outbound-poll`, + `whatsapp-message-actions`. +- Access-control coverage: `whatsapp-access-control-dm-open`, + `whatsapp-access-control-dm-disabled`, `whatsapp-access-control-group-open`, + `whatsapp-access-control-group-disabled`, `whatsapp-group-allowlist-block`. +- Native approvals: `whatsapp-approval-exec-deny-native`, + `whatsapp-approval-exec-native`, `whatsapp-approval-exec-reaction-native`, + `whatsapp-approval-plugin-native`. +- Status reactions: `whatsapp-status-reactions`. + +The catalog currently contains 35 scenarios. The `live-frontier` default lane is +kept small at 8 scenarios for fast smoke coverage. The `mock-openai` default +lane runs 29 deterministic scenarios through the real WhatsApp transport while +mocking only model output. Approval scenarios and a few heavier/blocking checks +remain explicit by scenario id. + +The WhatsApp QA driver observes structured live events (`text`, `media`, +`location`, `reaction`, and `poll`) and can actively send media, polls, +contacts, locations, and stickers. QA Lab imports that driver through the +`@openclaw/whatsapp/api.js` package surface instead of reaching into private +WhatsApp runtime files. Message content is redacted by default. Outbound +poll and upload-file coverage run through deterministic gateway `poll` and +`message.action` calls instead of model-prompt-only tool invocation. Output artifacts: diff --git a/extensions/qa-lab/src/live-transports/shared/live-artifacts.test.ts b/extensions/qa-lab/src/live-transports/shared/live-artifacts.test.ts index b3f2075c07d7..df8076720d56 100644 --- a/extensions/qa-lab/src/live-transports/shared/live-artifacts.test.ts +++ b/extensions/qa-lab/src/live-transports/shared/live-artifacts.test.ts @@ -1,8 +1,14 @@ // Qa Lab tests cover live artifacts plugin behavior. import { describe, expect, it } from "vitest"; -import { redactQaLiveLaneIssues } from "./live-artifacts.js"; +import { redactQaLiveLaneDetails, redactQaLiveLaneIssues } from "./live-artifacts.js"; describe("live transport artifacts", () => { + it("uses a stable public metadata redaction marker", () => { + expect(redactQaLiveLaneDetails()).toBe( + "details redacted (OPENCLAW_QA_REDACT_PUBLIC_METADATA=1)", + ); + }); + it("preserves cleanup phase labels while redacting details", () => { expect( redactQaLiveLaneIssues([ @@ -14,4 +20,17 @@ describe("live transport artifacts", () => { "live gateway cleanup: details redacted (OPENCLAW_QA_REDACT_PUBLIC_METADATA=1)", ]); }); + + it("redacts multi-line artifact errors without preserving later section labels", () => { + expect( + redactQaLiveLaneIssues([ + [ + "WhatsApp QA failed before scenario completion.", + "raw startup error with +15550000002", + "Artifacts:", + "- gatewayDebug: /tmp/openclaw-whatsapp-qa/gateway-debug", + ].join("\n"), + ]), + ).toEqual(["details redacted (OPENCLAW_QA_REDACT_PUBLIC_METADATA=1)"]); + }); }); diff --git a/extensions/qa-lab/src/live-transports/shared/live-artifacts.ts b/extensions/qa-lab/src/live-transports/shared/live-artifacts.ts index f75bcc823314..059402d1caf8 100644 --- a/extensions/qa-lab/src/live-transports/shared/live-artifacts.ts +++ b/extensions/qa-lab/src/live-transports/shared/live-artifacts.ts @@ -4,17 +4,20 @@ import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; const REDACTED_QA_LIVE_LANE_ISSUE_DETAILS = "details redacted (OPENCLAW_QA_REDACT_PUBLIC_METADATA=1)"; +export function redactQaLiveLaneDetails() { + return REDACTED_QA_LIVE_LANE_ISSUE_DETAILS; +} + export function appendQaLiveLaneIssue(issues: string[], label: string, error: unknown) { issues.push(`${label}: ${formatErrorMessage(error)}`); } export function redactQaLiveLaneIssues(issues: readonly string[]) { return issues.map((issue) => { - const separatorIndex = issue.indexOf(":"); - const label = separatorIndex < 0 ? "" : issue.slice(0, separatorIndex).trim(); - return label - ? `${label}: ${REDACTED_QA_LIVE_LANE_ISSUE_DETAILS}` - : REDACTED_QA_LIVE_LANE_ISSUE_DETAILS; + const firstLine = issue.split(/\r?\n/u, 1)[0] ?? ""; + const separatorIndex = firstLine.indexOf(":"); + const label = separatorIndex < 0 ? "" : firstLine.slice(0, separatorIndex).trim(); + return label ? `${label}: ${redactQaLiveLaneDetails()}` : redactQaLiveLaneDetails(); }); } diff --git a/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.test.ts b/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.test.ts index a8920711f681..9abb8887cc0a 100644 --- a/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.test.ts +++ b/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.test.ts @@ -108,9 +108,16 @@ describe("live transport scenario helpers", () => { standardId: "thread-follow-up", scenarioId: "slack-thread-follow-up", }); + expect(lanes.find((lane) => lane.transportId === "whatsapp")?.members).toContainEqual({ + standardId: "allowlist-block", + scenarioId: "whatsapp-group-allowlist-block", + }); expect( lanes.find((lane) => lane.transportId === "discord")?.baselineMissingStandardScenarioIds, ).toEqual(["allowlist-block", "top-level-reply-shape", "restart-resume"]); + expect( + lanes.find((lane) => lane.transportId === "whatsapp")?.baselineMissingStandardScenarioIds, + ).toEqual([]); }); it("keeps coverage report lane summaries aligned with runtime lanes", () => { diff --git a/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.ts b/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.ts index 748fa7c66f6b..79c17362b53b 100644 --- a/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.ts +++ b/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.ts @@ -72,8 +72,12 @@ export const LIVE_TRANSPORT_COVERAGE_LANES: readonly LiveTransportCoverageLane[] commandName: "whatsapp", members: [ { standardId: "canary", scenarioId: "whatsapp-canary" }, - { standardId: "allowlist-block", scenarioId: "whatsapp-pairing-block" }, { standardId: "mention-gating", scenarioId: "whatsapp-mention-gating" }, + { standardId: "top-level-reply-shape", scenarioId: "whatsapp-top-level-reply-shape" }, + { standardId: "restart-resume", scenarioId: "whatsapp-restart-resume" }, + { standardId: "help-command", scenarioId: "whatsapp-help-command" }, + { standardId: "reaction-observation", scenarioId: "whatsapp-status-reactions" }, + { standardId: "allowlist-block", scenarioId: "whatsapp-group-allowlist-block" }, ], }, ] as const; diff --git a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.test.ts b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.test.ts index b86f9a8ab54a..c74b07419604 100644 --- a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.test.ts @@ -4,7 +4,8 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { promisify } from "node:util"; -import { describe, expect, it } from "vitest"; +import type { WhatsAppQaDriverSession } from "@openclaw/whatsapp/api.js"; +import { describe, expect, it, vi } from "vitest"; import { testing } from "./whatsapp-live.runtime.js"; const execFileAsync = promisify(execFile); @@ -22,6 +23,62 @@ async function createTgz(params: { entries: Record; root: string return await fs.readFile(archivePath, "base64"); } +function createGatewayTargetContext(params: { gatewayTarget: string }) { + const calls: Array<{ method: string; payload: Record }> = []; + const context = { + gateway: { + call: async (method: string, payload: Record) => { + calls.push({ method, payload }); + return {}; + }, + }, + gatewayTarget: params.gatewayTarget, + scenarioId: "whatsapp-reply-context-isolation", + sutAccountId: "sut", + } satisfies Parameters[0]; + return { calls, context }; +} + +function createDiagnosticsContext( + messages: Array<{ + fromPhoneE164: string | null; + kind: "media" | "poll" | "reaction" | "text" | "unknown"; + messageId?: string; + observedAt: string; + quoted?: { messageId?: string; text?: string }; + text: string; + }>, +) { + return { + driver: { + getObservedMessages: () => messages, + }, + sutPhoneE164: "+15550000002", + } satisfies Parameters[0]; +} + +function createWhatsAppQaDriverMock( + overrides: Partial = {}, +): WhatsAppQaDriverSession { + return { + close: async () => {}, + getObservedMessages: () => [], + sendContact: async () => ({}), + sendLocation: async () => ({}), + sendMedia: async () => ({}), + sendPoll: async () => ({}), + sendReaction: async () => ({}), + sendSticker: async () => ({}), + sendText: async () => ({}), + waitForMessage: async () => ({ + kind: "text", + observedAt: new Date().toISOString(), + text: "ok", + }), + ...overrides, + }; +} + describe("WhatsApp QA live runtime", () => { it("parses credential payloads and normalizes phone numbers", () => { const payload = testing.parseWhatsAppQaCredentialPayload({ @@ -56,6 +113,7 @@ describe("WhatsApp QA live runtime", () => { { fromJid: "15550000002@s.whatsapp.net", fromPhoneE164: "+15550000002", + kind: "text", matchedScenario: true, messageId: "msg-1", observedAt: "2026-05-04T12:00:00.000Z", @@ -67,6 +125,7 @@ describe("WhatsApp QA live runtime", () => { }), ).toEqual([ { + kind: "text", matchedScenario: true, observedAt: "2026-05-04T12:00:00.000Z", scenarioId: "whatsapp-canary", @@ -83,6 +142,7 @@ describe("WhatsApp QA live runtime", () => { messages: [ { fromPhoneE164: "+15550000002", + kind: "text", observedAt: "2026-05-04T12:00:00.000Z", text: "captured body", }, @@ -90,12 +150,84 @@ describe("WhatsApp QA live runtime", () => { }), ).toEqual([ { + kind: "text", observedAt: "2026-05-04T12:00:00.000Z", text: "captured body", }, ]); }); + it("does not expose quoted message text when only metadata capture is enabled", () => { + expect( + testing.toObservedWhatsAppArtifacts({ + includeContent: false, + redactMetadata: false, + messages: [ + { + fromPhoneE164: "+15550000002", + kind: "text", + messageId: "msg-1", + observedAt: "2026-05-04T12:00:00.000Z", + quoted: { + messageId: "quoted-msg-1", + participant: "15550000001@s.whatsapp.net", + text: "quoted secret body", + }, + text: "secret body", + }, + ], + }), + ).toEqual([ + { + fromPhoneE164: "+15550000002", + kind: "text", + messageId: "msg-1", + observedAt: "2026-05-04T12:00:00.000Z", + quoted: { + messageId: "quoted-msg-1", + participant: "15550000001@s.whatsapp.net", + text: undefined, + }, + }, + ]); + }); + + it("does not expose reaction emoji when content capture is disabled", () => { + expect( + testing.toObservedWhatsAppArtifacts({ + includeContent: false, + redactMetadata: false, + messages: [ + { + fromPhoneE164: "+15550000002", + kind: "reaction", + messageId: "reaction-msg-1", + observedAt: "2026-05-04T12:00:00.000Z", + reaction: { + emoji: "👍", + fromMe: false, + messageId: "target-msg-1", + participant: "15550000001@s.whatsapp.net", + }, + text: "👍", + }, + ], + }), + ).toEqual([ + { + fromPhoneE164: "+15550000002", + kind: "reaction", + messageId: "reaction-msg-1", + observedAt: "2026-05-04T12:00:00.000Z", + reaction: { + fromMe: false, + messageId: "target-msg-1", + participant: "15550000001@s.whatsapp.net", + }, + }, + ]); + }); + it("derives a stable non-secret credential fingerprint", () => { expect(testing.fingerprintWhatsAppCredentialId("cred-stale-row")).toMatch( /^sha256:[0-9a-f]{16}$/, @@ -123,6 +255,49 @@ describe("WhatsApp QA live runtime", () => { expect(report).not.toContain("+15550000002"); }); + it("redacts published scenario details before rendering public artifacts", () => { + const publishedScenarios = testing.redactWhatsAppQaScenarioResults([ + { + id: "whatsapp-reply-delivery-shape", + title: "WhatsApp gateway send chunks long replies", + status: "pass", + details: "long reply chunked across raw-message-id-1 and raw-message-id-2", + }, + { + id: "whatsapp-inbound-structured-messages", + title: "WhatsApp inbound structured messages reach the agent", + status: "fail", + details: + "timed out waiting for WhatsApp QA driver message; observed 2 WhatsApp driver message(s) after wait lower bound: #1 observedAt=2026-06-04T23:47:00.000Z fromPhone=present kind=text textLength=17 messageId=present(length=10) quoted=missing quotedMessageId=missing fromExpectedSut=yes containsExpectedToken=no; #2 observedAt=2026-06-04T23:47:01.000Z fromPhone=present kind=text textLength=24 messageId=present(length=10) quoted=missing quotedMessageId=missing fromExpectedSut=no containsExpectedToken=yes", + }, + ]); + const report = testing.renderWhatsAppQaMarkdown({ + cleanupIssues: [ + "temporary auth cleanup failed: details redacted (OPENCLAW_QA_REDACT_PUBLIC_METADATA=1)", + ], + credentialSource: "convex", + finishedAt: "2026-05-04T12:01:00.000Z", + redactMetadata: true, + scenarios: publishedScenarios, + startedAt: "2026-05-04T12:00:00.000Z", + sutPhoneE164: "+15550000002", + }); + + expect(publishedScenarios[0]?.details).toBe( + "details redacted (OPENCLAW_QA_REDACT_PUBLIC_METADATA=1)", + ); + expect(publishedScenarios[1]?.details).toContain("observed 2 WhatsApp driver message(s)"); + expect(publishedScenarios[1]?.details).toContain("fromExpectedSut=yes"); + expect(publishedScenarios[1]?.details).toContain("textLength=17"); + expect(report).toContain("Details: details redacted"); + expect(report).toContain("observed 2 WhatsApp driver message(s)"); + expect(report).toContain("fromExpectedSut=yes"); + expect(report).toContain("textLength=17"); + expect(report).not.toContain("raw-message-id-1"); + expect(report).not.toContain("raw-message-id-2"); + expect(report).not.toContain("+15550000002"); + }); + it("unpacks auth archives into a caller-provided temp directory", async () => { const tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-wa-qa-test-")); try { @@ -160,36 +335,598 @@ describe("WhatsApp QA live runtime", () => { it("reports standard WhatsApp live transport scenario coverage", () => { expect(testing.WHATSAPP_QA_STANDARD_SCENARIO_IDS).toEqual([ "canary", - "allowlist-block", "mention-gating", + "top-level-reply-shape", + "restart-resume", + "help-command", + "reaction-observation", + "allowlist-block", ]); }); - it("keeps native approval scenarios out of default WhatsApp selection", () => { + it("uses opposite DM peers for driver sends and Gateway sends", () => { + expect( + testing.resolveWhatsAppQaMessageTargets({ + driverPhoneE164: "+15550000001", + scenarioTarget: "dm", + sutPhoneE164: "+15550000002", + }), + ).toEqual({ + driverTarget: "+15550000002", + gatewayTarget: "+15550000001", + }); + expect( + testing.resolveWhatsAppQaMessageTargets({ + driverPhoneE164: "+15550000001", + groupJid: "120363000000000000@g.us", + scenarioTarget: "group", + sutPhoneE164: "+15550000002", + }), + ).toEqual({ + driverTarget: "120363000000000000@g.us", + gatewayTarget: "120363000000000000@g.us", + }); + }); + + it("routes WhatsApp Gateway DM helper calls to the driver peer", async () => { + const { calls, context } = createGatewayTargetContext({ + gatewayTarget: "+15550000001", + }); + + await testing.callWhatsAppGatewaySend(context, { + label: "quoted", + message: "WHATSAPP_QA_QUOTED", + replyToId: "driver-message-1", + }); + await testing.callWhatsAppGatewayPoll(context, { + label: "poll", + options: ["alpha", "beta"], + question: "WHATSAPP_QA_POLL", + }); + await testing.callWhatsAppGatewayMessageAction(context, { + action: "react", + label: "react", + params: { + emoji: "👍", + messageId: "driver-message-1", + }, + }); + + expect(calls).toHaveLength(3); + expect(calls[0]?.payload).toMatchObject({ to: "+15550000001" }); + expect(calls[1]?.payload).toMatchObject({ to: "+15550000001" }); + expect(calls[2]?.payload.params).toMatchObject({ + emoji: "👍", + messageId: "driver-message-1", + to: "+15550000001", + }); + }); + + it("formats redacted wait diagnostics for unmatched WhatsApp observations", () => { + const diagnostics = testing.formatWhatsAppScenarioWaitDiagnostics( + createDiagnosticsContext([ + { + fromPhoneE164: "+15550000002", + kind: "text", + messageId: "before-lower-bound", + observedAt: "2026-06-05T00:59:59.000Z", + text: "SECRET_BEFORE", + }, + { + fromPhoneE164: "+15550000002", + kind: "text", + messageId: "fresh-message-secret-id", + observedAt: "2026-06-05T01:00:01.000Z", + quoted: { messageId: "quoted-secret-id", text: "quoted secret body" }, + text: "SECRET_MARKER", + }, + { + fromPhoneE164: "+15550000003", + kind: "media", + messageId: "other-sender-secret-id", + observedAt: "2026-06-05T01:00:02.000Z", + text: "SECRET_OTHER", + }, + ]), + { + diagnosticChecks: [ + { + label: "textMarker", + match: (message) => message.text.includes("SECRET_MARKER"), + }, + { + label: "quoteMatchesTrigger", + match: (message) => message.quoted?.messageId === "trigger-message", + }, + ], + observedAfter: new Date("2026-06-05T01:00:00.000Z"), + }, + ); + + expect(diagnostics).toContain("observed 2 WhatsApp driver message(s)"); + expect(diagnostics).toContain("fromExpectedSut=yes"); + expect(diagnostics).toContain("fromExpectedSut=no"); + expect(diagnostics).toContain("textMarker=yes"); + expect(diagnostics).toContain("quoteMatchesTrigger=no"); + expect(diagnostics).toContain("quoted=present"); + expect(diagnostics).toContain("quotedMessageId=present(length=16)"); + expect(diagnostics).not.toContain("+15550000002"); + expect(diagnostics).not.toContain("SECRET_MARKER"); + expect(diagnostics).not.toContain("fresh-message-secret-id"); + expect(diagnostics).not.toContain("quoted-secret-id"); + }); + + it("formats batch count diagnostics without exposing WhatsApp message content", () => { + const diagnostics = testing.formatWhatsAppBatchMessageDiagnostics([ + { + fromPhoneE164: "+15550000002", + kind: "text", + messageId: "batch-secret-id", + observedAt: "2026-06-05T01:00:01.000Z", + quoted: { messageId: "quoted-secret-id", text: "quoted secret body" }, + text: "SECRET_BATCH_BODY", + }, + ]); + + expect(diagnostics).toContain("textLength=17"); + expect(diagnostics).toContain("messageId=present(length=15)"); + expect(diagnostics).toContain("quoted=present"); + expect(diagnostics).not.toContain("+15550000002"); + expect(diagnostics).not.toContain("SECRET_BATCH_BODY"); + expect(diagnostics).not.toContain("batch-secret-id"); + expect(diagnostics).not.toContain("quoted secret body"); + }); + + it("treats any fresh SUT message as unexpected for no-reply scenarios", () => { + const unexpected = testing.findUnexpectedWhatsAppNoReplyMessage({ + messages: [ + { + fromPhoneE164: "+15550000002", + kind: "text", + observedAt: "2026-06-05T00:59:59.000Z", + text: "old generic access warning", + }, + { + fromPhoneE164: "+15550000003", + kind: "text", + observedAt: "2026-06-05T01:00:01.000Z", + text: "unrelated peer message", + }, + { + fromPhoneE164: "+15550000002", + kind: "text", + observedAt: "2026-06-05T01:00:02.000Z", + text: "generic access warning without the scenario marker", + }, + ], + observedAfter: new Date("2026-06-05T01:00:00.000Z"), + sutPhoneE164: "+15550000002", + target: "dm", + }); + + expect(unexpected?.text).toBe("generic access warning without the scenario marker"); + }); + + it("treats any fresh group message as unexpected for group no-reply scenarios", () => { + const unexpected = testing.findUnexpectedWhatsAppNoReplyMessage({ + groupJid: "120363000000000000@g.us", + messages: [ + { + fromJid: "120363111111111111@g.us", + fromPhoneE164: null, + kind: "text", + observedAt: "2026-06-05T01:00:01.000Z", + text: "different group message", + }, + { + fromJid: "120363000000000000@g.us", + fromPhoneE164: null, + kind: "text", + observedAt: "2026-06-05T01:00:02.000Z", + text: "generic group access warning without the scenario marker", + }, + ], + observedAfter: new Date("2026-06-05T01:00:00.000Z"), + sutPhoneE164: "+15550000002", + target: "group", + }); + + expect(unexpected?.text).toBe("generic group access warning without the scenario marker"); + }); + + it("keeps mock-backed and native approval scenarios out of default live-frontier selection", () => { const expectedDefaultIds = [ "whatsapp-canary", "whatsapp-pairing-block", "whatsapp-mention-gating", + "whatsapp-top-level-reply-shape", + "whatsapp-restart-resume", + "whatsapp-help-command", + "whatsapp-status-reactions", + "whatsapp-group-allowlist-block", ]; - expect(testing.findScenarios().map(({ id }) => id)).toEqual(expectedDefaultIds); - expect(testing.findScenarios([]).map(({ id }) => id)).toEqual(expectedDefaultIds); + expect(testing.findScenarios(undefined, "live-frontier").map(({ id }) => id)).toEqual( + expectedDefaultIds, + ); + expect(testing.findScenarios([], "live-frontier").map(({ id }) => id)).toEqual( + expectedDefaultIds, + ); + }); + + it("adds deterministic audio preflight to the default mock-openai WhatsApp selection", () => { + expect(testing.findScenarios(undefined, "mock-openai").map(({ id }) => id)).toEqual([ + "whatsapp-canary", + "whatsapp-pairing-block", + "whatsapp-mention-gating", + "whatsapp-top-level-reply-shape", + "whatsapp-restart-resume", + "whatsapp-help-command", + "whatsapp-commands-command", + "whatsapp-tools-compact-command", + "whatsapp-whoami-command", + "whatsapp-context-command", + "whatsapp-tool-only-usage-footer", + "whatsapp-reply-context-isolation", + "whatsapp-inbound-image-caption", + "whatsapp-audio-preflight", + "whatsapp-outbound-media-matrix", + "whatsapp-outbound-document-preserves-filename", + "whatsapp-outbound-poll", + "whatsapp-message-actions", + "whatsapp-inbound-structured-messages", + "whatsapp-group-audio-gating", + "whatsapp-access-control-dm-open", + "whatsapp-access-control-dm-disabled", + "whatsapp-access-control-group-open", + "whatsapp-access-control-group-disabled", + "whatsapp-reply-delivery-shape", + "whatsapp-stream-final-message-accounting", + "whatsapp-native-new-command", + "whatsapp-status-reactions", + "whatsapp-group-allowlist-block", + ]); + }); + + it("seeds the structured-message location check through text context", () => { + const [scenario] = testing.findScenarios(["whatsapp-inbound-structured-messages"]); + if (!scenario) { + throw new Error("missing structured WhatsApp scenario"); + } + const run = scenario.buildRun(); + if (run.kind === "approval") { + throw new Error("structured WhatsApp scenario unexpectedly built an approval run"); + } + + expect(run.input).toContain("37.774900, -122.419400"); + expect(run.input).toContain("WhatsApp location marker"); + expect(run.input).toContain("WhatsApp contact marker"); + expect(run.input).toContain("WhatsApp sticker marker"); + expect(run.input).toContain("exact marker before structured inbound checks"); + }); + + it("sends a WhatsApp-routable contact card in the structured-message check", async () => { + const sendContact = vi.fn(async () => ({ messageId: "contact-1" })); + const driver = createWhatsAppQaDriverMock({ + sendContact, + sendLocation: vi.fn(async () => ({ messageId: "location-1" })), + sendMedia: vi.fn(async () => ({ messageId: "document-1" })), + sendSticker: vi.fn(async () => ({ messageId: "sticker-1" })), + }); + + await testing.runWhatsAppStructuredInboundChecks({ + contactToken: "CONTACT_TOKEN", + documentToken: "DOCUMENT_TOKEN", + driver, + driverPhoneE164: "+15550000001", + locationToken: "LOCATION_TOKEN", + stickerToken: "STICKER_TOKEN", + target: "+15550000002", + waitForStructuredReply: async () => {}, + }); + + expect(sendContact).toHaveBeenCalledWith( + "+15550000002", + expect.objectContaining({ + vcard: expect.stringContaining("waid=15550000001:+15550000001"), + }), + ); + }); + + it("labels structured-message contact wait failures", async () => { + const sendSticker = vi.fn(async () => ({ messageId: "sticker-1" })); + const driver = createWhatsAppQaDriverMock({ + sendContact: vi.fn(async () => ({ messageId: "contact-1" })), + sendLocation: vi.fn(async () => ({ messageId: "location-1" })), + sendMedia: vi.fn(async () => ({ messageId: "document-1" })), + sendSticker, + }); + + await expect( + testing.runWhatsAppStructuredInboundChecks({ + contactToken: "CONTACT_TOKEN", + documentToken: "DOCUMENT_TOKEN", + driver, + driverPhoneE164: "+15550000001", + locationToken: "LOCATION_TOKEN", + stickerToken: "STICKER_TOKEN", + target: "+15550000002", + waitForStructuredReply: async (label, _observedAfter, expectedToken) => { + if (label === "contact") { + throw new Error( + `timed out waiting for WhatsApp structured ${label} reply (${expectedToken})`, + ); + } + }, + }), + ).rejects.toThrow("timed out waiting for WhatsApp structured contact reply"); + expect(sendSticker).not.toHaveBeenCalled(); + }); + + it("formats approval wait diagnostics without exposing message content", () => { + const observedAfter = new Date("2026-06-05T18:36:57.000Z"); + const diagnostics = testing.formatWhatsAppApprovalWaitDiagnostics({ + approvalId: "plugin:approval-1", + approvalKind: "plugin", + driver: createWhatsAppQaDriverMock({ + getObservedMessages: () => [ + { + fromPhoneE164: "+15550000002", + kind: "text", + messageId: "message-1", + observedAt: "2026-06-05T18:36:58.000Z", + text: "unrelated text that should not be copied into diagnostics", + }, + ], + }), + observedAfter, + state: "pending", + sutPhoneE164: "+15550000002", + token: "TOKEN-1", + }); + + expect(diagnostics).toContain("observed 1 WhatsApp driver message(s)"); + expect(diagnostics).toContain("fromExpectedSut=yes"); + expect(diagnostics).toContain("approvalText=no"); + expect(diagnostics).toContain("messageId=present(length=9)"); + expect(diagnostics).not.toContain("unrelated text"); + }); + + it("formats per-scenario progress lines for live lane visibility", () => { + const [scenario] = testing.findScenarios(["whatsapp-inbound-structured-messages"]); + if (!scenario) { + throw new Error("missing structured WhatsApp scenario"); + } + + expect( + testing.formatWhatsAppScenarioProgressLine({ + details: "timed out waiting for WhatsApp QA driver message", + index: 21, + scenario, + status: "fail", + total: 35, + }), + ).toBe( + "[whatsapp-qa] [21/35] fail whatsapp-inbound-structured-messages: " + + "WhatsApp inbound structured messages reach the agent - " + + "timed out waiting for WhatsApp QA driver message", + ); + }); + + it("redacts per-scenario progress details when public metadata redaction is enabled", () => { + expect( + testing.formatWhatsAppScenarioProgressDetails({ + details: "long reply chunked across raw-message-id-1 and raw-message-id-2", + redactMetadata: true, + }), + ).toBe("details redacted (OPENCLAW_QA_REDACT_PUBLIC_METADATA=1)"); + expect( + testing.formatWhatsAppScenarioProgressDetails({ + details: + "timed out waiting for WhatsApp QA driver message; observed 1 WhatsApp driver message(s) after wait lower bound: #1 observedAt=2026-06-04T23:47:00.000Z fromPhone=present kind=text textLength=17 messageId=present(length=10) quoted=missing quotedMessageId=missing fromExpectedSut=yes", + redactMetadata: true, + }), + ).toBe( + "observed 1 WhatsApp driver message(s) after wait lower bound: " + + "#1 observedAt=2026-06-04T23:47:00.000Z fromPhone=present kind=text " + + "textLength=17 messageId=present(length=10) quoted=missing " + + "quotedMessageId=missing fromExpectedSut=yes", + ); + expect( + testing.formatWhatsAppScenarioProgressDetails({ + details: "safe local diagnostic", + redactMetadata: false, + }), + ).toBe("safe local diagnostic"); + }); + + it("adds WhatsApp command UX parity scenarios to the mock-backed selection", () => { + const scenarios = testing.findScenarios([ + "whatsapp-commands-command", + "whatsapp-tools-compact-command", + "whatsapp-whoami-command", + "whatsapp-context-command", + "whatsapp-tool-only-usage-footer", + ]); + + expect( + scenarios.map((scenario) => { + const run = scenario.buildRun(); + if (run.kind === "approval") { + throw new Error(`${scenario.id} unexpectedly built an approval run`); + } + return [ + scenario.id, + run.input, + String(run.matchText), + run.expectedJoinedSutTextIncludes, + run.expectedSutMessageCountRange, + ] as const; + }), + ).toEqual([ + [ + "whatsapp-commands-command", + "/commands", + "/Commands \\(|\\/session|\\/verbose/iu", + ["/session", "/verbose"], + undefined, + ], + [ + "whatsapp-tools-compact-command", + "/tools compact", + "/Available tools|exec|Use \\/tools verbose for descriptions/iu", + ["exec", "Use /tools verbose for descriptions"], + undefined, + ], + [ + "whatsapp-whoami-command", + "/whoami", + "/(?=.*Identity)(?=.*Channel: whatsapp)(?=.*AllowFrom:)/isu", + undefined, + undefined, + ], + [ + "whatsapp-context-command", + "/context list", + "/(?=.*Context breakdown)(?=.*Workspace:)(?=.*Tool schemas)/isu", + undefined, + undefined, + ], + [ + "whatsapp-tool-only-usage-footer", + "/usage tokens", + "/Usage footer: tokens/iu", + undefined, + undefined, + ], + ]); + expect(scenarios.map((scenario) => scenario.defaultProviderModes)).toEqual([ + ["mock-openai"], + ["mock-openai"], + ["mock-openai"], + ["mock-openai"], + ["mock-openai"], + ]); + }); + + it("defines WhatsApp final-message accounting as a settled two-chunk assertion", () => { + const [scenario] = testing.findScenarios(["whatsapp-stream-final-message-accounting"]); + const run = scenario.buildRun(); + if (run.kind === "approval") { + throw new Error("whatsapp-stream-final-message-accounting unexpectedly built approval run"); + } + + expect(scenario.defaultProviderModes).toEqual(["mock-openai"]); + expect(run.input).toContain("WhatsApp long final QA check"); + expect(run.matchText).toBe("WHATSAPP-LONG-FINAL-BEGIN"); + expect(run.expectedJoinedSutTextIncludes).toEqual([ + "WHATSAPP-LONG-FINAL-BEGIN", + "WHATSAPP-LONG-FINAL-END", + ]); + expect(run.expectedSutMessageCount).toBe(2); + expect(run.settleMs).toBe(4_000); + }); + + it("requires the long-reply delivery-shape tail marker in the second chunk", async () => { + const [scenario] = testing.findScenarios(["whatsapp-reply-delivery-shape"]); + const run = scenario.buildRun(); + if (run.kind === "approval" || !run.afterReply) { + throw new Error("whatsapp-reply-delivery-shape unexpectedly omitted afterReply"); + } + const token = String(run.matchText); + let waitCallCount = 0; + const driver = createWhatsAppQaDriverMock({ + waitForMessage: async (params) => { + waitCallCount += 1; + if (waitCallCount === 1) { + const firstChunk = { + fromPhoneE164: "+15550000002", + kind: "text" as const, + messageId: "chunk-1", + observedAt: "2026-06-05T01:00:01.000Z", + text: `${token}_LONG_BEGIN`, + }; + expect(params.match(firstChunk)).toBe(true); + return firstChunk; + } + + const missingTailMarker = { + fromPhoneE164: "+15550000002", + kind: "text" as const, + messageId: "chunk-2", + observedAt: "2026-06-05T01:00:02.000Z", + text: "second chunk without the tail marker", + }; + const tailChunk = { + fromPhoneE164: "+15550000002", + kind: "text" as const, + messageId: "chunk-3", + observedAt: "2026-06-05T01:00:03.000Z", + text: `${token}_LONG_END`, + }; + expect(params.match(missingTailMarker)).toBe(false); + expect(params.match(tailChunk)).toBe(true); + return tailChunk; + }, + }); + const context = { + driver, + driverPhoneE164: "+15550000001", + gateway: { + call: async () => ({}), + restart: async () => {}, + workspaceDir: "/tmp/openclaw-whatsapp-qa-gateway", + }, + gatewayTarget: "+15550000001", + gatewayWorkspaceDir: "/tmp/openclaw-whatsapp-qa-gateway", + recordObservedMessage: () => {}, + requestStartedAt: new Date("2026-06-05T01:00:00.000Z"), + scenarioId: "whatsapp-reply-delivery-shape", + scenarioTitle: "WhatsApp gateway send chunks long replies", + sent: { messageId: "driver-message-1" }, + sutAccountId: "sut", + sutPhoneE164: "+15550000002", + target: "+15550000002", + waitForReady: async () => {}, + } satisfies Parameters>[1]; + + await run.afterReply( + { + fromPhoneE164: "+15550000002", + kind: "text", + messageId: "initial-reply", + observedAt: "2026-06-05T01:00:00.500Z", + text: token, + }, + context, + ); + + expect(waitCallCount).toBe(2); }); it("selects native approval scenarios by id without changing standard coverage", () => { const scenarios = testing.findScenarios([ "whatsapp-approval-exec-native", + "whatsapp-approval-exec-reaction-native", "whatsapp-approval-plugin-native", ]); expect(scenarios.map(({ id }) => id)).toEqual([ "whatsapp-approval-exec-native", + "whatsapp-approval-exec-reaction-native", "whatsapp-approval-plugin-native", ]); expect(testing.WHATSAPP_QA_STANDARD_SCENARIO_IDS).not.toContain( "whatsapp-approval-exec-native", ); - expect(scenarios.map((scenario) => scenario.buildRun().kind)).toEqual(["approval", "approval"]); + expect(scenarios.map((scenario) => scenario.buildRun().kind)).toEqual([ + "approval", + "approval", + "approval", + ]); + expect(scenarios[1]?.buildRun()).toMatchObject({ + decisionMode: "reaction", + }); }); it("enables WhatsApp native exec and plugin approval delivery for approval scenarios", () => { @@ -216,6 +953,140 @@ describe("WhatsApp QA live runtime", () => { expect(account).not.toHaveProperty("execApprovals"); }); + it("enables WhatsApp audio preflight with the OpenAI transcription provider", () => { + const cfg = testing.buildWhatsAppQaConfig( + {}, + { + allowFrom: ["+15550000001"], + authDir: "/tmp/openclaw-whatsapp-qa-auth", + dmPolicy: "allowlist", + overrides: { + audioPreflight: true, + }, + sutAccountId: "sut", + }, + ); + + expect(cfg.plugins?.allow).toContain("whatsapp"); + expect(cfg.tools?.media?.audio).toEqual({ + enabled: true, + models: [{ provider: "openai", model: "gpt-4o-transcribe" }], + }); + }); + + it("enables WhatsApp action discovery for message action scenarios", () => { + const cfg = testing.buildWhatsAppQaConfig( + {}, + { + allowFrom: ["+15550000001"], + authDir: "/tmp/openclaw-whatsapp-qa-auth", + dmPolicy: "allowlist", + overrides: { + actions: true, + }, + sutAccountId: "sut", + }, + ); + + expect(cfg.channels?.whatsapp?.actions).toEqual({ reactions: true, polls: true }); + expect(cfg.channels?.whatsapp?.reactionLevel).toBe("minimal"); + }); + + it("defines the WhatsApp audio preflight scenario as mock-backed audio media", () => { + const [scenario] = testing.findScenarios(["whatsapp-audio-preflight"]); + const scenarioRun = scenario.buildRun(); + if (scenarioRun.kind === "approval") { + throw new Error("whatsapp-audio-preflight unexpectedly built an approval scenario run"); + } + + expect(scenario.requiredPluginIds).toEqual(["openai"]); + expect(scenario.defaultProviderModes).toEqual(["mock-openai"]); + expect(scenarioRun.expectReply).toBe(true); + expect(scenarioRun.matchText).toBe("WHATSAPP_QA_AUDIO_TRANSCRIPT_OK"); + expect(scenarioRun.sendMode).toMatchObject({ + fileName: "whatsapp-qa-audio.wav", + kind: "media", + mediaType: "audio/wav", + }); + expect(scenarioRun.sendMode?.kind === "media" && scenarioRun.sendMode.mediaBuffer.length).toBe( + 32_044, + ); + }); + + it("defines group audio gating as captionless audio driven by mock transcription", () => { + const [scenario] = testing.findScenarios(["whatsapp-group-audio-gating"]); + const scenarioRun = scenario.buildRun(); + if (scenarioRun.kind === "approval") { + throw new Error("whatsapp-group-audio-gating unexpectedly built an approval scenario run"); + } + + expect(scenarioRun.input).toBe(""); + expect(scenarioRun.matchText).toBe("WHATSAPP_QA_GROUP_AUDIO_TRANSCRIPT_OK"); + expect(scenarioRun.quietInput).toBe(""); + expect(scenarioRun.quietMatchText).toBeUndefined(); + expect(scenarioRun.sendMode).toMatchObject({ + fileName: "whatsapp-qa-group-audio.wav", + kind: "media", + mediaType: "audio/wav", + }); + expect(scenarioRun.quietSendMode).toMatchObject({ + fileName: "whatsapp-qa-group-audio-quiet.wav", + kind: "media", + mediaType: "audio/wav", + }); + expect( + scenarioRun.sendMode?.kind === "media" && + scenarioRun.quietSendMode?.kind === "media" && + scenarioRun.sendMode.mediaBuffer.length > scenarioRun.quietSendMode.mediaBuffer.length, + ).toBe(true); + }); + + it("applies WhatsApp QA config overrides for reply mode and status reactions", () => { + const cfg = testing.buildWhatsAppQaConfig( + {}, + { + allowFrom: ["+15550000001"], + authDir: "/tmp/openclaw-whatsapp-qa-auth", + dmPolicy: "allowlist", + overrides: { + replyToMode: "all", + statusReactions: true, + }, + sutAccountId: "sut", + }, + ); + + expect(cfg.channels?.whatsapp?.accounts?.sut?.replyToMode).toBe("all"); + expect(cfg.channels?.whatsapp?.ackReaction).toMatchObject({ + direct: true, + emoji: "👀", + }); + expect(cfg.messages?.statusReactions?.enabled).toBe(true); + }); + + it("can configure a group scenario as sender allowlist-blocked instead of open mention-gated", () => { + const cfg = testing.buildWhatsAppQaConfig( + {}, + { + allowFrom: ["+15550000000"], + authDir: "/tmp/openclaw-whatsapp-qa-auth", + dmPolicy: "allowlist", + groupJid: "120363000000000000@g.us", + overrides: { + blockGroupSender: true, + groupPolicy: "allowlist", + }, + sutAccountId: "sut", + }, + ); + + const account = cfg.channels?.whatsapp?.accounts?.sut; + expect(account?.groupPolicy).toBe("allowlist"); + expect(account?.groupAllowFrom).toEqual(["+15550000001"]); + expect(account?.groupAllowFrom).not.toContain("+15550000000"); + expect(account?.groups).toBeUndefined(); + }); + it("matches native approval resolved text emitted by the WhatsApp approval handler", () => { expect( testing.matchesWhatsAppApprovalResolvedText({ @@ -231,6 +1102,22 @@ describe("WhatsApp QA live runtime", () => { text: "✅ Plugin approval allowed once. ID: whatsapp-qa-plugin-123", }), ).toBe(true); + expect( + testing.matchesWhatsAppApprovalResolvedText({ + approvalId: "whatsapp-qa-exec-deny-123", + approvalKind: "exec", + decision: "deny", + text: "✅ Exec approval deny. ID: whatsapp-qa-exec-deny-123", + }), + ).toBe(true); + expect( + testing.matchesWhatsAppApprovalResolvedText({ + approvalId: "whatsapp-qa-plugin-deny-123", + approvalKind: "plugin", + decision: "deny", + text: "✅ Plugin approval denied. ID: whatsapp-qa-plugin-deny-123", + }), + ).toBe(true); }); it("uses automatic visible replies for WhatsApp group mention gating", () => { @@ -310,9 +1197,14 @@ describe("WhatsApp QA live runtime", () => { ); expect( testing.isTransientWhatsAppQaDriverError( - new Error("timed out waiting for WhatsApp QA driver message"), + new Error("timed out after 45000ms waiting for WhatsApp QA driver pending notifications"), ), ).toBe(true); + expect( + testing.isTransientWhatsAppQaDriverError( + new Error("timed out waiting for WhatsApp QA driver message"), + ), + ).toBe(false); expect(testing.isTransientWhatsAppQaDriverError(new Error("timed out waiting"))).toBe(false); }); }); diff --git a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.ts b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.ts index 2189b79a8c89..9e23860a2733 100644 --- a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.ts +++ b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.ts @@ -4,7 +4,11 @@ import { randomUUID } from "node:crypto"; import fs from "node:fs/promises"; import path from "node:path"; import { promisify } from "node:util"; -import { startWhatsAppQaDriverSession } from "@openclaw/whatsapp/api.js"; +import { + startWhatsAppQaDriverSession, + type WhatsAppQaDriverObservedMessage, + type WhatsAppQaDriverSession, +} from "@openclaw/whatsapp/api.js"; import { normalizeE164 } from "openclaw/plugin-sdk/account-resolution"; import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; @@ -17,6 +21,7 @@ import { fingerprintQaCredentialId } from "../../qa-credentials-fingerprint.runt import { defaultQaModelForMode, normalizeQaProviderMode, + type QaProviderMode, type QaProviderModeInput, } from "../../run-config.js"; import { @@ -27,6 +32,8 @@ import { import { appendQaLiveLaneIssue as appendLiveLaneIssue, buildQaLiveLaneArtifactsError as buildLiveLaneArtifactsError, + redactQaLiveLaneDetails, + redactQaLiveLaneIssues, } from "../shared/live-artifacts.js"; import { startQaLiveLaneGateway } from "../shared/live-gateway.runtime.js"; import { @@ -46,30 +53,139 @@ export type WhatsAppQaRuntimeEnv = { }; type WhatsAppQaScenarioId = + | "whatsapp-access-control-dm-disabled" + | "whatsapp-access-control-dm-open" + | "whatsapp-access-control-group-disabled" + | "whatsapp-access-control-group-open" + | "whatsapp-approval-exec-deny-native" + | "whatsapp-approval-exec-reaction-native" + | "whatsapp-audio-preflight" | "whatsapp-canary" + | "whatsapp-commands-command" + | "whatsapp-context-command" + | "whatsapp-group-allowlist-block" + | "whatsapp-group-audio-gating" + | "whatsapp-help-command" + | "whatsapp-inbound-image-caption" + | "whatsapp-inbound-structured-messages" + | "whatsapp-message-actions" + | "whatsapp-native-new-command" + | "whatsapp-outbound-document-preserves-filename" + | "whatsapp-outbound-media-matrix" + | "whatsapp-outbound-poll" | "whatsapp-pairing-block" | "whatsapp-mention-gating" + | "whatsapp-reply-delivery-shape" + | "whatsapp-reply-context-isolation" + | "whatsapp-reply-to-message" + | "whatsapp-restart-resume" + | "whatsapp-stream-final-message-accounting" + | "whatsapp-status-command" + | "whatsapp-status-reactions" + | "whatsapp-top-level-reply-shape" + | "whatsapp-tools-compact-command" + | "whatsapp-tool-only-usage-footer" + | "whatsapp-whoami-command" | "whatsapp-approval-exec-native" | "whatsapp-approval-plugin-native"; type WhatsAppQaApprovalKind = "exec" | "plugin"; -type WhatsAppQaApprovalDecision = "allow-once"; +type WhatsAppQaApprovalDecision = "allow-once" | "deny"; +type WhatsAppQaApprovalDecisionMode = "reaction" | "rpc"; + +type WhatsAppQaMessageSendMode = + | { + kind?: "text"; + } + | { + fileName?: string; + kind: "media"; + mediaBuffer: Buffer; + mediaType: string; + }; + +type WhatsAppQaGateway = Awaited>; +type WhatsAppQaGatewayRuntime = Pick; +type WhatsAppQaGatewayCallContext = { + gateway: Pick; + gatewayTarget: string; + scenarioId: WhatsAppQaScenarioId; + sutAccountId: string; +}; +type WhatsAppQaObservedMessagesContext = { + driver: Pick; + sutPhoneE164: string; +}; + +type WhatsAppQaMessageScenarioContext = { + driver: WhatsAppQaDriverSession; + driverPhoneE164: string; + gateway: WhatsAppQaGatewayRuntime; + gatewayTarget: string; + gatewayWorkspaceDir: string; + recordObservedMessage: (message: WhatsAppQaDriverObservedMessage) => void; + requestStartedAt: Date; + scenarioId: WhatsAppQaScenarioId; + scenarioTitle: string; + sent: { messageId?: string }; + sutAccountId: string; + sutPhoneE164: string; + target: string; + waitForReady: () => Promise; +}; + +function resolveWhatsAppQaMessageTargets(params: { + driverPhoneE164: string; + groupJid?: string; + scenarioTarget: "dm" | "group"; + sutPhoneE164: string; +}) { + if (params.scenarioTarget === "group") { + if (!params.groupJid) { + throw new Error("WhatsApp group scenario requires groupJid."); + } + return { + driverTarget: params.groupJid, + gatewayTarget: params.groupJid, + }; + } + return { + driverTarget: params.sutPhoneE164, + gatewayTarget: params.driverPhoneE164, + }; +} type WhatsAppQaMessageScenarioRun = { - configMode: "allowlist" | "pairing"; + afterReply?: ( + reply: WhatsAppQaDriverObservedMessage, + context: WhatsAppQaMessageScenarioContext, + ) => Promise | string | undefined; + afterSend?: (context: WhatsAppQaMessageScenarioContext) => Promise; + configMode: "allowlist" | "disabled" | "open" | "pairing"; expectReply: boolean; + expectedJoinedSutTextIncludes?: string[]; + expectedSutMessageCount?: number; + expectedSutMessageCountRange?: readonly [number, number]; input: string; kind?: "message"; matchText: string | RegExp; quietInput?: string; quietMatchText?: string | RegExp; + quietSendMode?: WhatsAppQaMessageSendMode; quietWindowMs?: number; + sendMode?: WhatsAppQaMessageSendMode; + settleMs?: number; target: "dm" | "group"; + verify?: ( + reply: WhatsAppQaDriverObservedMessage, + context: WhatsAppQaMessageScenarioContext, + ) => void; }; type WhatsAppQaApprovalScenarioRun = { approvalKind: WhatsAppQaApprovalKind; decision: WhatsAppQaApprovalDecision; + decisionMode?: WhatsAppQaApprovalDecisionMode; kind: "approval"; token: string; }; @@ -77,56 +193,59 @@ type WhatsAppQaApprovalScenarioRun = { type WhatsAppQaScenarioRun = WhatsAppQaApprovalScenarioRun | WhatsAppQaMessageScenarioRun; type WhatsAppQaConfigOverrides = { + actions?: boolean; + audioPreflight?: boolean; approvals?: { exec?: boolean; plugin?: boolean; }; + blockGroupSender?: boolean; + groupPolicy?: "allowlist" | "disabled" | "open"; + replyToMode?: "all" | "first" | "off"; + statusReactions?: boolean; }; type WhatsAppQaScenarioDefinition = LiveTransportScenarioDefinition & { buildRun: () => WhatsAppQaScenarioRun; configOverrides?: WhatsAppQaConfigOverrides; + defaultEnabled?: boolean; + defaultProviderModes?: readonly QaProviderMode[]; requiresGroupJid?: boolean; + requiredPluginIds?: readonly string[]; }; -type WhatsAppQaDriverObservedMessage = { - fromJid?: string; - fromPhoneE164?: string | null; - messageId?: string; - observedAt: string; - text: string; -}; - -type WhatsAppQaDriverSession = { - close: () => Promise; - getObservedMessages: () => WhatsAppQaDriverObservedMessage[]; - sendText: (to: string, text: string) => Promise<{ messageId?: string }>; - waitForMessage: (params: { - match: (message: WhatsAppQaDriverObservedMessage) => boolean; - timeoutMs: number; - }) => Promise; -}; - -type WhatsAppQaGateway = Awaited>; - -type WhatsAppObservedMessage = WhatsAppQaDriverObservedMessage & { +interface WhatsAppObservedMessage extends WhatsAppQaDriverObservedMessage { approvalState?: "pending" | "resolved"; matchedScenario?: boolean; scenarioId?: string; scenarioTitle?: string; -}; +} type WhatsAppObservedMessageArtifact = { approvalState?: "pending" | "resolved"; fromPhoneE164?: string | null; + hasMedia?: boolean; + kind?: WhatsAppQaDriverObservedMessage["kind"]; matchedScenario?: boolean; + mediaFileName?: string; + mediaType?: string; messageId?: string; observedAt: string; + poll?: WhatsAppQaDriverObservedMessage["poll"]; + quoted?: WhatsAppQaDriverObservedMessage["quoted"]; + reaction?: WhatsAppObservedReactionArtifact; scenarioId?: string; scenarioTitle?: string; text?: string; }; +type WhatsAppObservedReactionArtifact = { + emoji?: string; + fromMe?: boolean; + messageId?: string; + participant?: string; +}; + type WhatsAppQaScenarioResult = { details: string; id: string; @@ -193,6 +312,119 @@ const WHATSAPP_QA_ENV_KEYS = [ "OPENCLAW_QA_WHATSAPP_DRIVER_AUTH_ARCHIVE_BASE64", "OPENCLAW_QA_WHATSAPP_SUT_AUTH_ARCHIVE_BASE64", ] as const; +const WHATSAPP_QA_ONE_PIXEL_PNG = Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAFgwJ/lzK4ZQAAAABJRU5ErkJggg==", + "base64", +); +const WHATSAPP_QA_ONE_PIXEL_WEBP = Buffer.from( + "UklGRiIAAABXRUJQVlA4IBYAAAAwAQCdASoBAAEADsD+JaQAA3AA/vuUAAA=", + "base64", +); +const WHATSAPP_QA_AUDIO_TRANSCRIPT_MARKER = "WHATSAPP_QA_AUDIO_TRANSCRIPT_OK"; +const WHATSAPP_QA_GROUP_AUDIO_TRANSCRIPT_MARKER = "WHATSAPP_QA_GROUP_AUDIO_TRANSCRIPT_OK"; + +function createWhatsAppQaPdfBuffer() { + return Buffer.from( + [ + "%PDF-1.4", + "1 0 obj", + "<< /Type /Catalog /Pages 2 0 R >>", + "endobj", + "2 0 obj", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + "endobj", + "3 0 obj", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] >>", + "endobj", + "trailer", + "<< /Root 1 0 R >>", + "%%EOF", + "", + ].join("\n"), + "utf8", + ); +} + +type WhatsAppStructuredInboundDriver = Pick< + WhatsAppQaDriverSession, + "sendContact" | "sendLocation" | "sendMedia" | "sendSticker" +>; + +async function runWhatsAppStructuredInboundChecks(params: { + contactToken: string; + documentToken: string; + driver: WhatsAppStructuredInboundDriver; + driverPhoneE164: string; + locationToken: string; + stickerToken: string; + target: string; + waitForStructuredReply: ( + label: string, + observedAfter: Date, + expectedToken: string, + ) => Promise; +}) { + const documentStartedAt = new Date(); + await params.driver.sendMedia( + params.target, + `Reply with only this exact marker after reading the document caption: ${params.documentToken}`, + createWhatsAppQaPdfBuffer(), + "application/pdf", + { fileName: "whatsapp-qa-document.pdf" }, + ); + await params.waitForStructuredReply("document", documentStartedAt, params.documentToken); + + const locationStartedAt = new Date(); + await params.driver.sendLocation(params.target, { + degreesLatitude: 37.7749, + degreesLongitude: -122.4194, + }); + await params.waitForStructuredReply("location", locationStartedAt, params.locationToken); + + const contactStartedAt = new Date(); + const driverContactWaId = params.driverPhoneE164.replace(/\D/g, ""); + await params.driver.sendContact(params.target, { + displayName: "WhatsApp QA Driver Contact", + vcard: [ + "BEGIN:VCARD", + "VERSION:3.0", + "FN:WhatsApp QA Driver Contact", + `TEL;type=CELL;type=VOICE;waid=${driverContactWaId}:${params.driverPhoneE164}`, + "END:VCARD", + ].join("\n"), + }); + await params.waitForStructuredReply("contact", contactStartedAt, params.contactToken); + + const stickerStartedAt = new Date(); + await params.driver.sendSticker(params.target, WHATSAPP_QA_ONE_PIXEL_WEBP, { + mimetype: "image/webp", + }); + await params.waitForStructuredReply("sticker", stickerStartedAt, params.stickerToken); +} + +function createWhatsAppQaAudioWavBuffer(params?: { durationSeconds?: number }) { + const sampleRate = 16_000; + const channelCount = 1; + const bitsPerSample = 16; + const durationSeconds = params?.durationSeconds ?? 1; + const bytesPerSample = bitsPerSample / 8; + const dataBytes = sampleRate * durationSeconds * channelCount * bytesPerSample; + const buffer = Buffer.alloc(44 + dataBytes); + buffer.write("RIFF", 0, "ascii"); + buffer.writeUInt32LE(36 + dataBytes, 4); + buffer.write("WAVE", 8, "ascii"); + buffer.write("fmt ", 12, "ascii"); + buffer.writeUInt32LE(16, 16); + buffer.writeUInt16LE(1, 20); + buffer.writeUInt16LE(channelCount, 22); + buffer.writeUInt32LE(sampleRate, 24); + buffer.writeUInt32LE(sampleRate * channelCount * bytesPerSample, 28); + buffer.writeUInt16LE(channelCount * bytesPerSample, 32); + buffer.writeUInt16LE(bitsPerSample, 34); + buffer.write("data", 36, "ascii"); + buffer.writeUInt32LE(dataBytes, 40); + return buffer; +} const whatsappQaCredentialPayloadSchema = z.object({ driverPhoneE164: z.string().trim().min(1), @@ -221,8 +453,8 @@ const WHATSAPP_QA_SCENARIOS: WhatsAppQaScenarioDefinition[] = [ }, { id: "whatsapp-pairing-block", - standardId: "allowlist-block", title: "WhatsApp non-allowlisted DM gets pairing gate", + defaultProviderModes: ["live-frontier", "mock-openai"], timeoutMs: 20_000, buildRun: () => ({ configMode: "pairing", @@ -253,6 +485,870 @@ const WHATSAPP_QA_SCENARIOS: WhatsAppQaScenarioDefinition[] = [ }; }, }, + { + id: "whatsapp-top-level-reply-shape", + standardId: "top-level-reply-shape", + title: "WhatsApp DM top-level reply shape", + timeoutMs: 60_000, + configOverrides: { + replyToMode: "off", + }, + buildRun: () => { + const token = `WHATSAPP_QA_TOP_LEVEL_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + configMode: "allowlist", + expectReply: true, + input: `Reply with only this exact marker: ${token}`, + matchText: token, + target: "dm", + verify: (reply) => { + if (reply.quoted?.messageId) { + throw new Error( + `expected top-level WhatsApp reply without quote metadata, got quoted message ${reply.quoted.messageId}`, + ); + } + }, + }; + }, + }, + { + id: "whatsapp-restart-resume", + standardId: "restart-resume", + title: "WhatsApp DM resumes after gateway restart", + timeoutMs: 120_000, + buildRun: () => { + const firstToken = `WHATSAPP_QA_RESTART_BEFORE_${randomUUID().slice(0, 8).toUpperCase()}`; + const secondToken = `WHATSAPP_QA_RESTART_AFTER_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + afterReply: async (_reply, context) => { + await context.gateway.restart(); + await context.waitForReady(); + const secondStartedAt = new Date(); + await context.driver.sendText( + context.target, + `After the restart, reply with only this exact marker: ${secondToken}`, + ); + await context.driver.waitForMessage({ + observedAfter: secondStartedAt, + timeoutMs: 60_000, + match: (message) => + message.fromPhoneE164 === context.sutPhoneE164 && message.text.includes(secondToken), + }); + return "gateway restarted and post-restart reply matched"; + }, + configMode: "allowlist", + expectReply: true, + input: `Before the restart, reply with only this exact marker: ${firstToken}`, + matchText: firstToken, + target: "dm", + }; + }, + }, + { + id: "whatsapp-help-command", + standardId: "help-command", + title: "WhatsApp help command replies", + timeoutMs: 60_000, + buildRun: () => ({ + configMode: "allowlist", + expectReply: true, + input: "/help", + matchText: /OpenClaw|commands|status|\/new/iu, + target: "dm", + }), + }, + { + id: "whatsapp-status-command", + title: "WhatsApp status command replies", + timeoutMs: 60_000, + buildRun: () => ({ + configMode: "allowlist", + expectReply: true, + input: "/status", + matchText: /OpenClaw|status|session|agent/iu, + target: "dm", + }), + }, + { + id: "whatsapp-commands-command", + title: "WhatsApp commands list replies", + defaultProviderModes: ["mock-openai"], + timeoutMs: 60_000, + buildRun: () => ({ + configMode: "allowlist", + expectReply: true, + expectedJoinedSutTextIncludes: ["/session", "/verbose"], + input: "/commands", + matchText: /Commands \(|\/session|\/verbose/iu, + settleMs: 4_000, + target: "dm", + }), + }, + { + id: "whatsapp-tools-compact-command", + title: "WhatsApp tools compact reply", + defaultProviderModes: ["mock-openai"], + timeoutMs: 60_000, + buildRun: () => ({ + configMode: "allowlist", + expectReply: true, + expectedJoinedSutTextIncludes: ["exec", "Use /tools verbose for descriptions"], + input: "/tools compact", + matchText: /Available tools|exec|Use \/tools verbose for descriptions/iu, + settleMs: 4_000, + target: "dm", + }), + }, + { + id: "whatsapp-whoami-command", + title: "WhatsApp whoami reply", + defaultProviderModes: ["mock-openai"], + timeoutMs: 60_000, + buildRun: () => ({ + configMode: "allowlist", + expectReply: true, + input: "/whoami", + matchText: /(?=.*Identity)(?=.*Channel: whatsapp)(?=.*AllowFrom:)/isu, + target: "dm", + }), + }, + { + id: "whatsapp-context-command", + title: "WhatsApp context list reply", + defaultProviderModes: ["mock-openai"], + timeoutMs: 60_000, + buildRun: () => ({ + configMode: "allowlist", + expectReply: true, + input: "/context list", + matchText: /(?=.*Context breakdown)(?=.*Workspace:)(?=.*Tool schemas)/isu, + target: "dm", + }), + }, + { + id: "whatsapp-tool-only-usage-footer", + title: "WhatsApp tool-only reply includes usage footer", + defaultProviderModes: ["mock-openai"], + timeoutMs: 120_000, + buildRun: () => { + const token = `WHATSAPP_QA_USAGE_FOOTER_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + afterReply: async (_reply, context) => { + const usageStartedAt = new Date(); + await context.driver.sendText( + context.target, + `Reply with only this exact marker after usage footer setup: ${token}`, + ); + const usageReply = await context.driver.waitForMessage({ + observedAfter: usageStartedAt, + timeoutMs: 60_000, + match: (message) => + message.fromPhoneE164 === context.sutPhoneE164 && + message.text.includes(token) && + message.text.includes("Usage:"), + }); + context.recordObservedMessage(usageReply); + return "model reply included visible usage footer"; + }, + configMode: "allowlist", + expectReply: true, + input: "/usage tokens", + matchText: /Usage footer: tokens/iu, + target: "dm", + }; + }, + }, + { + id: "whatsapp-reply-to-message", + title: "WhatsApp DM reply-to mode quotes the triggering message", + timeoutMs: 60_000, + configOverrides: { + replyToMode: "all", + }, + buildRun: () => { + const token = `WHATSAPP_QA_REPLY_TO_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + configMode: "allowlist", + expectReply: true, + input: `Reply with only this exact marker: ${token}`, + matchText: token, + target: "dm", + verify: (reply, context) => { + if (!context.sent.messageId) { + throw new Error("WhatsApp driver did not return a triggering message id."); + } + if (reply.quoted?.messageId !== context.sent.messageId) { + throw new Error( + `expected reply quote ${context.sent.messageId}, got ${reply.quoted?.messageId ?? ""}`, + ); + } + }, + }; + }, + }, + { + id: "whatsapp-reply-context-isolation", + title: "WhatsApp fresh gateway send does not reuse prior quote context", + defaultProviderModes: ["mock-openai"], + timeoutMs: 120_000, + buildRun: () => { + const token = `WHATSAPP_QA_REPLY_ISOLATION_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + afterReply: async (_reply, context) => { + if (!context.sent.messageId) { + throw new Error("WhatsApp driver did not return a triggering message id."); + } + const quotedStartedAt = new Date(); + await callWhatsAppGatewaySend(context, { + label: "quoted", + message: `${token}_QUOTED`, + replyToId: context.sent.messageId, + }); + await waitForScenarioObservedMessage(context, { + observedAfter: quotedStartedAt, + diagnosticChecks: [ + { + label: "textMarker", + match: (message) => message.text.includes(`${token}_QUOTED`), + }, + { + label: "quotedMessageIdMatchesTrigger", + match: (message) => message.quoted?.messageId === context.sent.messageId, + }, + ], + match: (message) => message.text.includes(`${token}_QUOTED`), + }); + + const freshStartedAt = new Date(); + await callWhatsAppGatewaySend(context, { + label: "fresh", + message: `${token}_FRESH`, + }); + const fresh = await waitForScenarioObservedMessage(context, { + observedAfter: freshStartedAt, + match: (message) => message.text.includes(`${token}_FRESH`), + }); + if (fresh.quoted?.messageId) { + throw new Error( + `expected fresh WhatsApp send without quote metadata, got quoted message ${fresh.quoted.messageId}`, + ); + } + return "quoted send and fresh send used independent reply context"; + }, + configMode: "allowlist", + expectReply: true, + input: `Reply with only this exact marker before reply isolation checks: ${token}`, + matchText: token, + target: "dm", + }; + }, + }, + { + id: "whatsapp-inbound-image-caption", + title: "WhatsApp inbound image caption reaches the agent", + defaultProviderModes: ["mock-openai"], + timeoutMs: 60_000, + buildRun: () => { + const token = `WHATSAPP_QA_IMAGE_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + configMode: "allowlist", + expectReply: true, + input: `This image caption asks you to reply with only this exact marker: ${token}`, + matchText: token, + sendMode: { + fileName: "whatsapp-qa.png", + kind: "media", + mediaBuffer: WHATSAPP_QA_ONE_PIXEL_PNG, + mediaType: "image/png", + }, + target: "dm", + }; + }, + }, + { + id: "whatsapp-audio-preflight", + title: "WhatsApp inbound audio preflight transcript reaches the agent", + defaultProviderModes: ["mock-openai"], + timeoutMs: 90_000, + configOverrides: { + audioPreflight: true, + }, + requiredPluginIds: ["openai"], + buildRun: () => ({ + configMode: "allowlist", + expectReply: true, + input: "", + matchText: WHATSAPP_QA_AUDIO_TRANSCRIPT_MARKER, + sendMode: { + fileName: "whatsapp-qa-audio.wav", + kind: "media", + mediaBuffer: createWhatsAppQaAudioWavBuffer(), + mediaType: "audio/wav", + }, + target: "dm", + }), + }, + { + id: "whatsapp-outbound-media-matrix", + title: "WhatsApp gateway send delivers outbound media variants", + defaultProviderModes: ["mock-openai"], + timeoutMs: 120_000, + buildRun: () => { + const token = `WHATSAPP_QA_OUTBOUND_MEDIA_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + afterReply: async (_reply, context) => { + const mediaRootToken = randomUUID().slice(0, 8); + const imagePath = await writeWhatsAppQaWorkspaceFixture(context, { + buffer: WHATSAPP_QA_ONE_PIXEL_PNG, + fileName: `whatsapp-qa-${mediaRootToken}.png`, + }); + const documentPath = await writeWhatsAppQaWorkspaceFixture(context, { + buffer: createWhatsAppQaPdfBuffer(), + fileName: `whatsapp-qa-${mediaRootToken}.pdf`, + }); + const audioPath = await writeWhatsAppQaWorkspaceFixture(context, { + buffer: createWhatsAppQaAudioWavBuffer(), + fileName: `whatsapp-qa-${mediaRootToken}.wav`, + }); + + const imageStartedAt = new Date(); + await callWhatsAppGatewaySend(context, { + label: "image", + mediaUrl: imagePath, + message: `${token}_IMAGE`, + }); + await waitForScenarioObservedMessage(context, { + observedAfter: imageStartedAt, + match: (message) => + message.kind === "media" && + message.hasMedia === true && + message.mediaType?.startsWith("image/") === true && + message.text.includes(`${token}_IMAGE`), + }); + + const documentStartedAt = new Date(); + await callWhatsAppGatewaySend(context, { + forceDocument: true, + label: "document", + mediaUrl: documentPath, + message: `${token}_DOCUMENT`, + }); + await waitForScenarioObservedMessage(context, { + observedAfter: documentStartedAt, + match: (message) => + message.kind === "media" && + message.hasMedia === true && + (message.mediaType === "application/pdf" || + message.mediaFileName?.endsWith(".pdf") === true) && + message.text.includes(`${token}_DOCUMENT`), + }); + + const audioStartedAt = new Date(); + await callWhatsAppGatewaySend(context, { + asVoice: true, + label: "audio", + mediaUrl: audioPath, + message: `${token}_AUDIO`, + }); + await waitForScenarioObservedMessage(context, { + observedAfter: audioStartedAt, + match: (message) => + message.kind === "media" && + message.hasMedia === true && + message.mediaType?.startsWith("audio/") === true, + }); + await waitForScenarioObservedMessage(context, { + observedAfter: audioStartedAt, + match: (message) => message.text.includes(`${token}_AUDIO`), + }); + + const multiStartedAt = new Date(); + await callWhatsAppGatewaySend(context, { + label: "multi", + mediaUrls: [imagePath, documentPath], + message: `${token}_MULTI`, + }); + await waitForScenarioObservedMessage(context, { + observedAfter: multiStartedAt, + match: (message) => + message.kind === "media" && message.mediaType?.startsWith("image/") === true, + }); + await waitForScenarioObservedMessage(context, { + observedAfter: multiStartedAt, + match: (message) => + message.kind === "media" && + (message.mediaType === "application/pdf" || + message.mediaFileName?.endsWith(".pdf") === true), + }); + return "gateway send delivered image, document, audio, and multi-media"; + }, + configMode: "allowlist", + expectReply: true, + input: `Reply with only this exact marker before outbound media checks: ${token}`, + matchText: token, + target: "dm", + }; + }, + }, + { + id: "whatsapp-outbound-document-preserves-filename", + title: "WhatsApp outbound document preserves filename and caption", + defaultProviderModes: ["mock-openai"], + timeoutMs: 90_000, + buildRun: () => { + const token = `WHATSAPP_QA_DOCUMENT_FILE_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + afterReply: async (_reply, context) => { + const documentPath = await writeWhatsAppQaWorkspaceFixture(context, { + buffer: createWhatsAppQaPdfBuffer(), + fileName: `whatsapp-qa-report-${token}.pdf`, + }); + const documentStartedAt = new Date(); + await callWhatsAppGatewaySend(context, { + forceDocument: true, + label: "document-filename", + mediaUrl: documentPath, + message: `${token}_CAPTION`, + }); + const document = await waitForScenarioObservedMessage(context, { + observedAfter: documentStartedAt, + match: (message) => + message.kind === "media" && + message.hasMedia === true && + message.text.includes(`${token}_CAPTION`) && + message.mediaFileName === `whatsapp-qa-report-${token}.pdf`, + }); + return `document ${document.mediaFileName ?? ""} preserved`; + }, + configMode: "allowlist", + expectReply: true, + input: `Reply with only this exact marker before document filename check: ${token}`, + matchText: token, + target: "dm", + }; + }, + }, + { + id: "whatsapp-outbound-poll", + title: "WhatsApp gateway poll delivers outbound native poll", + defaultProviderModes: ["mock-openai"], + timeoutMs: 90_000, + buildRun: () => { + const token = `WHATSAPP_QA_OUTBOUND_POLL_${randomUUID().slice(0, 8).toUpperCase()}`; + const question = `${token} choose one`; + return { + afterReply: async (_reply, context) => { + const pollStartedAt = new Date(); + await callWhatsAppGatewayPoll(context, { + label: "poll", + options: ["alpha", "beta"], + question, + }); + const poll = await waitForScenarioObservedMessage(context, { + observedAfter: pollStartedAt, + match: (message) => + message.kind === "poll" && + message.poll?.question === question && + message.poll.options.includes("alpha") && + message.poll.options.includes("beta"), + }); + return `poll observed with ${poll.poll?.options.length ?? 0} options`; + }, + configMode: "allowlist", + expectReply: true, + input: `Reply with only this exact marker before outbound poll check: ${token}`, + matchText: token, + target: "dm", + }; + }, + }, + { + id: "whatsapp-message-actions", + title: "WhatsApp message.action react and upload-file execute", + defaultProviderModes: ["mock-openai"], + timeoutMs: 120_000, + configOverrides: { + actions: true, + }, + buildRun: () => { + const token = `WHATSAPP_QA_ACTIONS_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + afterReply: async (_reply, context) => { + if (!context.sent.messageId) { + throw new Error("WhatsApp driver did not return a triggering message id."); + } + const reactionStartedAt = new Date(); + await callWhatsAppGatewayMessageAction(context, { + action: "react", + label: "react", + params: { + emoji: "👍", + messageId: context.sent.messageId, + }, + }); + await waitForScenarioObservedMessage(context, { + observedAfter: reactionStartedAt, + match: (message) => + message.kind === "reaction" && + message.reaction?.messageId === context.sent.messageId && + message.reaction?.emoji === "👍", + }); + + const uploadStartedAt = new Date(); + await callWhatsAppGatewayMessageAction(context, { + action: "upload-file", + label: "upload-file", + params: { + buffer: WHATSAPP_QA_ONE_PIXEL_PNG.toString("base64"), + caption: `${token}_UPLOAD`, + contentType: "image/png", + filename: "whatsapp-qa-upload.png", + }, + }); + await waitForScenarioObservedMessage(context, { + observedAfter: uploadStartedAt, + match: (message) => + message.kind === "media" && + message.mediaType?.startsWith("image/") === true && + message.text.includes(`${token}_UPLOAD`), + }); + return "message.action react and upload-file observed"; + }, + configMode: "allowlist", + expectReply: true, + input: `Reply with only this exact marker before action checks: ${token}`, + matchText: token, + target: "dm", + }; + }, + }, + { + id: "whatsapp-inbound-structured-messages", + title: "WhatsApp inbound structured messages reach the agent", + defaultProviderModes: ["mock-openai"], + timeoutMs: 240_000, + buildRun: () => { + const token = `WHATSAPP_QA_STRUCTURED_${randomUUID().slice(0, 8).toUpperCase()}`; + const locationToken = `${token}_LOCATION`; + const contactToken = `${token}_CONTACT`; + const stickerToken = `${token}_STICKER`; + const locationCoordinateText = "37.774900, -122.419400"; + return { + afterReply: async (_reply, context) => { + const waitForStructuredReply = async ( + label: string, + observedAfter: Date, + expectedToken: string, + ) => { + try { + return await waitForScenarioObservedMessage(context, { + observedAfter, + timeoutMs: 60_000, + match: (message) => message.text.includes(expectedToken), + diagnosticChecks: [ + { + label: "containsExpectedToken", + match: (message) => message.text.includes(expectedToken), + }, + ], + }); + } catch (error) { + throw new Error( + `timed out waiting for WhatsApp structured ${label} reply (${expectedToken}): ${formatErrorMessage(error)}`, + { cause: error }, + ); + } + }; + + await runWhatsAppStructuredInboundChecks({ + contactToken, + documentToken: `${token}_DOCUMENT`, + driver: context.driver, + driverPhoneE164: context.driverPhoneE164, + locationToken, + stickerToken, + target: context.target, + waitForStructuredReply, + }); + return "document, location, contact, and sticker elicited replies"; + }, + configMode: "allowlist", + expectReply: true, + input: + `When a later WhatsApp location message shows ${locationCoordinateText}, ` + + `reply with only this WhatsApp location marker: ${locationToken}. ` + + `When a later WhatsApp contact message appears, ` + + `reply with only this WhatsApp contact marker: ${contactToken}. ` + + `When a later WhatsApp sticker message appears, ` + + `reply with only this WhatsApp sticker marker: ${stickerToken}. ` + + `Reply with only this exact marker before structured inbound checks: ${token}`, + matchText: token, + target: "dm", + }; + }, + }, + { + id: "whatsapp-group-audio-gating", + title: "WhatsApp group audio mention gating", + defaultProviderModes: ["mock-openai"], + timeoutMs: 120_000, + configOverrides: { + audioPreflight: true, + }, + requiredPluginIds: ["openai"], + requiresGroupJid: true, + buildRun: () => ({ + configMode: "allowlist", + expectReply: true, + input: "", + matchText: WHATSAPP_QA_GROUP_AUDIO_TRANSCRIPT_MARKER, + quietInput: "", + quietSendMode: { + fileName: "whatsapp-qa-group-audio-quiet.wav", + kind: "media", + mediaBuffer: createWhatsAppQaAudioWavBuffer(), + mediaType: "audio/wav", + }, + quietWindowMs: 5_000, + sendMode: { + fileName: "whatsapp-qa-group-audio.wav", + kind: "media", + mediaBuffer: createWhatsAppQaAudioWavBuffer({ durationSeconds: 2 }), + mediaType: "audio/wav", + }, + target: "group", + }), + }, + { + id: "whatsapp-access-control-dm-open", + title: "WhatsApp dmPolicy open allows direct messages", + defaultProviderModes: ["mock-openai"], + timeoutMs: 60_000, + buildRun: () => { + const token = `WHATSAPP_QA_DM_OPEN_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + configMode: "open", + expectReply: true, + input: `Reply with only this exact marker under dmPolicy open: ${token}`, + matchText: token, + target: "dm", + }; + }, + }, + { + id: "whatsapp-access-control-dm-disabled", + title: "WhatsApp dmPolicy disabled stays quiet", + defaultProviderModes: ["mock-openai"], + timeoutMs: 8_000, + buildRun: () => { + const token = `WHATSAPP_QA_DM_DISABLED_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + configMode: "disabled", + expectReply: false, + input: `Do not reply under dmPolicy disabled. Forbidden marker: ${token}`, + matchText: token, + target: "dm", + }; + }, + }, + { + id: "whatsapp-access-control-group-open", + title: "WhatsApp groupPolicy open allows mention-gated groups", + defaultProviderModes: ["mock-openai"], + requiresGroupJid: true, + timeoutMs: 60_000, + configOverrides: { + groupPolicy: "open", + }, + buildRun: () => { + const token = `WHATSAPP_QA_GROUP_OPEN_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + configMode: "allowlist", + expectReply: true, + input: `openclawqa reply with only this exact marker under groupPolicy open: ${token}`, + matchText: token, + target: "group", + }; + }, + }, + { + id: "whatsapp-access-control-group-disabled", + title: "WhatsApp groupPolicy disabled stays quiet", + defaultProviderModes: ["mock-openai"], + requiresGroupJid: true, + timeoutMs: 8_000, + configOverrides: { + groupPolicy: "disabled", + }, + buildRun: () => { + const token = `WHATSAPP_QA_GROUP_DISABLED_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + configMode: "allowlist", + expectReply: false, + input: `openclawqa groupPolicy disabled must not reply with ${token}`, + matchText: token, + target: "group", + }; + }, + }, + { + id: "whatsapp-reply-delivery-shape", + title: "WhatsApp gateway send chunks long replies", + defaultProviderModes: ["mock-openai"], + timeoutMs: 120_000, + buildRun: () => { + const token = `WHATSAPP_QA_REPLY_SHAPE_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + afterReply: async (_reply, context) => { + if (!context.sent.messageId) { + throw new Error("WhatsApp driver did not return a triggering message id."); + } + const chunkStartedAt = new Date(); + const longText = `${token}_LONG_BEGIN\n${"A".repeat(4_500)}\n${token}_LONG_END`; + await callWhatsAppGatewaySend(context, { + label: "long-reply", + message: longText, + replyToId: context.sent.messageId, + }); + const firstChunk = await waitForScenarioObservedMessage(context, { + observedAfter: chunkStartedAt, + diagnosticChecks: [ + { + label: "longBeginMarker", + match: (message) => message.text.includes(`${token}_LONG_BEGIN`), + }, + ], + match: (message) => message.text.includes(`${token}_LONG_BEGIN`), + }); + const secondChunk = await waitForScenarioObservedMessage(context, { + observedAfter: chunkStartedAt, + diagnosticChecks: [ + { + label: "longEndMarker", + match: (message) => message.text.includes(`${token}_LONG_END`), + }, + ], + match: (message) => + message.messageId !== firstChunk.messageId && + message.text.includes(`${token}_LONG_END`), + }); + return `long reply chunked across ${firstChunk.messageId ?? ""} and ${secondChunk.messageId ?? ""}`; + }, + configMode: "allowlist", + expectReply: true, + input: `Reply with only this exact marker before reply-shape checks: ${token}`, + matchText: token, + target: "dm", + }; + }, + }, + { + id: "whatsapp-stream-final-message-accounting", + title: "WhatsApp streamed final response has exactly the final chunks", + defaultProviderModes: ["mock-openai"], + timeoutMs: 90_000, + buildRun: () => ({ + configMode: "allowlist", + expectReply: true, + expectedJoinedSutTextIncludes: ["WHATSAPP-LONG-FINAL-BEGIN", "WHATSAPP-LONG-FINAL-END"], + expectedSutMessageCount: 2, + input: "WhatsApp long final QA check. Use the scripted long final response.", + matchText: "WHATSAPP-LONG-FINAL-BEGIN", + settleMs: 4_000, + target: "dm", + }), + }, + { + id: "whatsapp-native-new-command", + title: "WhatsApp /new command starts a new session", + defaultProviderModes: ["mock-openai"], + timeoutMs: 60_000, + buildRun: () => ({ + configMode: "allowlist", + expectReply: true, + input: "/new", + matchText: /new session|session/i, + target: "dm", + }), + }, + { + id: "whatsapp-approval-exec-deny-native", + title: "WhatsApp native exec approval prompt denies", + timeoutMs: 60_000, + configOverrides: { + approvals: { + exec: true, + }, + }, + buildRun: () => ({ + approvalKind: "exec", + decision: "deny", + kind: "approval", + token: `WHATSAPP_QA_EXEC_DENY_${randomUUID().slice(0, 8).toUpperCase()}`, + }), + }, + { + id: "whatsapp-status-reactions", + standardId: "reaction-observation", + title: "WhatsApp status reactions are observable", + timeoutMs: 60_000, + configOverrides: { + statusReactions: true, + }, + buildRun: () => { + const token = `WHATSAPP_QA_STATUS_REACTION_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + afterSend: async (context) => { + if (!context.sent.messageId) { + throw new Error("WhatsApp driver did not return a triggering message id."); + } + const reaction = await context.driver.waitForMessage({ + observedAfter: context.requestStartedAt, + timeoutMs: 30_000, + match: (message) => { + const observedReaction = message.reaction; + if (!observedReaction) { + return false; + } + return ( + message.kind === "reaction" && + message.fromPhoneE164 === context.sutPhoneE164 && + observedReaction.messageId === context.sent.messageId && + Boolean(observedReaction.emoji) + ); + }, + }); + return `status reaction ${reaction.reaction?.emoji ?? ""} observed`; + }, + configMode: "allowlist", + expectReply: true, + input: `Reply with only this exact marker after normal processing: ${token}`, + matchText: token, + target: "dm", + }; + }, + }, + { + id: "whatsapp-group-allowlist-block", + standardId: "allowlist-block", + title: "WhatsApp group outside allowlist stays quiet", + timeoutMs: 8_000, + configOverrides: { + blockGroupSender: true, + groupPolicy: "allowlist", + }, + requiresGroupJid: true, + buildRun: () => { + const quietToken = `WHATSAPP_QA_GROUP_BLOCK_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + configMode: "allowlist", + expectReply: false, + input: `openclawqa blocked group should not reply with ${quietToken}`, + matchText: quietToken, + target: "group", + }; + }, + }, { id: "whatsapp-approval-exec-native", title: "WhatsApp native exec approval prompt resolves", @@ -269,6 +1365,23 @@ const WHATSAPP_QA_SCENARIOS: WhatsAppQaScenarioDefinition[] = [ token: `WHATSAPP_QA_EXEC_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`, }), }, + { + id: "whatsapp-approval-exec-reaction-native", + title: "WhatsApp native exec approval resolves from reaction", + timeoutMs: 60_000, + configOverrides: { + approvals: { + exec: true, + }, + }, + buildRun: () => ({ + approvalKind: "exec", + decision: "allow-once", + decisionMode: "reaction", + kind: "approval", + token: `WHATSAPP_QA_EXEC_REACTION_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`, + }), + }, { id: "whatsapp-approval-plugin-native", title: "WhatsApp native plugin approval prompt resolves with exec approvals enabled", @@ -372,12 +1485,29 @@ function parseWhatsAppQaCredentialPayload(payload: unknown): WhatsAppQaRuntimeEn return validateWhatsAppQaRuntimeEnv(parsed, "WhatsApp credential payload"); } -function defaultWhatsAppQaScenarios() { - return WHATSAPP_QA_SCENARIOS.filter((scenario) => scenario.standardId); +function shouldRunWhatsAppScenarioByDefault( + scenario: WhatsAppQaScenarioDefinition, + providerMode: QaProviderMode, +) { + if (scenario.defaultEnabled === false) { + return false; + } + if (scenario.standardId) { + return true; + } + return Boolean(scenario.defaultProviderModes?.includes(providerMode)); } -function findScenarios(ids?: string[]) { - const scenarios = ids && ids.length > 0 ? WHATSAPP_QA_SCENARIOS : defaultWhatsAppQaScenarios(); +function findScenarios( + ids?: string[], + providerMode: QaProviderMode = DEFAULT_QA_LIVE_PROVIDER_MODE, +) { + const scenarios = + ids && ids.length > 0 + ? WHATSAPP_QA_SCENARIOS + : WHATSAPP_QA_SCENARIOS.filter((scenario) => + shouldRunWhatsAppScenarioByDefault(scenario, providerMode), + ); return selectLiveTransportScenarios({ ids, laneLabel: "WhatsApp", @@ -385,12 +1515,27 @@ function findScenarios(ids?: string[]) { }); } +function buildNonMatchingWhatsAppQaAllowFrom(existingAllowFrom: string[]) { + const existing = new Set( + existingAllowFrom + .map((value) => normalizeE164(value)) + .filter((value): value is string => Boolean(value)), + ); + for (let suffix = 0; suffix <= 9999; suffix += 1) { + const candidate = `+1555${String(suffix).padStart(7, "0")}`; + if (!existing.has(candidate)) { + return [candidate]; + } + } + throw new Error("Unable to derive a WhatsApp QA groupAllowFrom entry outside allowFrom."); +} + function buildWhatsAppQaConfig( baseCfg: OpenClawConfig, params: { allowFrom: string[]; authDir: string; - dmPolicy: "allowlist" | "pairing"; + dmPolicy: "allowlist" | "disabled" | "open" | "pairing"; groupJid?: string; overrides?: WhatsAppQaConfigOverrides; sutAccountId: string; @@ -398,6 +1543,30 @@ function buildWhatsAppQaConfig( ): OpenClawConfig { const pluginAllow = uniqueStrings([...(baseCfg.plugins?.allow ?? []), "whatsapp"]); const approvalOverrides = params.overrides?.approvals; + const groupPolicy = params.overrides?.groupPolicy ?? "open"; + const groupAllowFrom = params.overrides?.blockGroupSender + ? buildNonMatchingWhatsAppQaAllowFrom(params.allowFrom) + : undefined; + const audioPreflightConfig = params.overrides?.audioPreflight + ? { + tools: { + ...baseCfg.tools, + media: { + ...baseCfg.tools?.media, + audio: { + ...baseCfg.tools?.media?.audio, + enabled: true, + models: [ + { + provider: "openai", + model: "gpt-4o-transcribe", + }, + ], + }, + }, + }, + } + : {}; const approvalForwardingConfig = approvalOverrides?.exec || approvalOverrides?.plugin ? { @@ -427,6 +1596,7 @@ function buildWhatsAppQaConfig( return { ...baseCfg, ...approvalForwardingConfig, + ...audioPreflightConfig, plugins: { ...baseCfg.plugins, allow: pluginAllow, @@ -440,38 +1610,82 @@ function buildWhatsAppQaConfig( whatsapp: { enabled: true, defaultAccount: params.sutAccountId, + ...(params.overrides?.statusReactions + ? { + ackReaction: { + ...baseCfg.channels?.whatsapp?.ackReaction, + direct: true, + emoji: "👀", + }, + } + : {}), + ...(params.overrides?.actions + ? { + actions: { + reactions: true, + polls: true, + }, + reactionLevel: "minimal" as const, + } + : {}), accounts: { [params.sutAccountId]: { enabled: true, authDir: params.authDir, dmPolicy: params.dmPolicy, allowFrom: params.allowFrom, + ...(params.overrides?.replyToMode + ? { + replyToMode: params.overrides.replyToMode, + } + : {}), ...(params.groupJid ? { - groupPolicy: "open" as const, - groups: { - [params.groupJid]: { requireMention: true }, - }, + groupPolicy, + ...(groupAllowFrom + ? { + groupAllowFrom, + } + : {}), + ...(groupPolicy === "open" + ? { + groups: { + [params.groupJid]: { requireMention: true }, + }, + } + : {}), } : {}), }, }, }, }, - ...(params.groupJid + ...(params.groupJid || params.overrides?.statusReactions ? { messages: { ...baseCfg.messages, - groupChat: { - ...baseCfg.messages?.groupChat, - visibleReplies: "automatic", - mentionPatterns: [ - ...new Set([ - ...(baseCfg.messages?.groupChat?.mentionPatterns ?? []), - "\\bopenclawqa\\b", - ]), - ], - }, + ...(params.groupJid + ? { + groupChat: { + ...baseCfg.messages?.groupChat, + visibleReplies: "automatic", + mentionPatterns: [ + ...new Set([ + ...(baseCfg.messages?.groupChat?.mentionPatterns ?? []), + "\\bopenclawqa\\b", + ]), + ], + }, + } + : {}), + ...(params.overrides?.statusReactions + ? { + statusReactions: { + ...baseCfg.messages?.statusReactions, + enabled: true, + }, + } + : {}), }, } : {}), @@ -607,13 +1821,312 @@ function messageMatches(message: WhatsAppObservedMessage, matchText: string | Re : matchText.test(message.text); } +function buildWhatsAppQaIdempotencyKey(scenarioId: WhatsAppQaScenarioId, label: string) { + return `${scenarioId}:${label}:${randomUUID()}`; +} + +async function writeWhatsAppQaWorkspaceFixture( + context: WhatsAppQaMessageScenarioContext, + params: { + buffer: Buffer; + fileName: string; + }, +) { + const fixtureDir = path.join(context.gatewayWorkspaceDir, ".openclaw", "qa-whatsapp-media"); + await fs.mkdir(fixtureDir, { recursive: true }); + const filePath = path.join(fixtureDir, params.fileName); + await fs.writeFile(filePath, params.buffer); + return filePath; +} + +async function callWhatsAppGatewaySend( + context: WhatsAppQaGatewayCallContext, + params: { + asVoice?: boolean; + forceDocument?: boolean; + label: string; + mediaUrl?: string; + mediaUrls?: string[]; + message?: string; + replyToId?: string; + }, +) { + return await context.gateway.call( + "send", + { + accountId: context.sutAccountId, + agentId: "main", + channel: "whatsapp", + idempotencyKey: buildWhatsAppQaIdempotencyKey(context.scenarioId, params.label), + to: context.gatewayTarget, + ...(params.message !== undefined ? { message: params.message } : {}), + ...(params.mediaUrl ? { mediaUrl: params.mediaUrl } : {}), + ...(params.mediaUrls ? { mediaUrls: params.mediaUrls } : {}), + ...(params.asVoice !== undefined ? { asVoice: params.asVoice } : {}), + ...(params.forceDocument !== undefined ? { forceDocument: params.forceDocument } : {}), + ...(params.replyToId ? { replyToId: params.replyToId } : {}), + }, + { timeoutMs: 60_000 }, + ); +} + +async function callWhatsAppGatewayPoll( + context: WhatsAppQaGatewayCallContext, + params: { + label: string; + maxSelections?: number; + options: string[]; + question: string; + }, +) { + return await context.gateway.call( + "poll", + { + accountId: context.sutAccountId, + channel: "whatsapp", + idempotencyKey: buildWhatsAppQaIdempotencyKey(context.scenarioId, params.label), + maxSelections: params.maxSelections, + options: params.options, + question: params.question, + to: context.gatewayTarget, + }, + { timeoutMs: 60_000 }, + ); +} + +async function callWhatsAppGatewayMessageAction( + context: WhatsAppQaGatewayCallContext, + params: { + action: "react" | "upload-file"; + label: string; + params: Record; + }, +) { + return await context.gateway.call( + "message.action", + { + accountId: context.sutAccountId, + action: params.action, + channel: "whatsapp", + idempotencyKey: buildWhatsAppQaIdempotencyKey(context.scenarioId, params.label), + params: { + ...params.params, + to: context.gatewayTarget, + }, + }, + { timeoutMs: 60_000 }, + ); +} + +async function waitForScenarioObservedMessage( + context: WhatsAppQaMessageScenarioContext, + params: { + diagnosticChecks?: Array<{ + label: string; + match: (message: WhatsAppQaDriverObservedMessage) => boolean; + }>; + match: (message: WhatsAppQaDriverObservedMessage) => boolean; + observedAfter?: Date; + timeoutMs?: number; + }, +) { + let message: WhatsAppQaDriverObservedMessage; + try { + message = await context.driver.waitForMessage({ + observedAfter: params.observedAfter, + timeoutMs: params.timeoutMs ?? 45_000, + match: (candidate) => + candidate.fromPhoneE164 === context.sutPhoneE164 && params.match(candidate), + }); + } catch (error) { + if (/\btimed out waiting for WhatsApp QA driver message\b/iu.test(formatErrorMessage(error))) { + throw new Error( + `${formatErrorMessage(error)}; ${formatWhatsAppScenarioWaitDiagnostics(context, { + diagnosticChecks: params.diagnosticChecks, + observedAfter: params.observedAfter, + })}`, + { cause: error }, + ); + } + throw error; + } + context.recordObservedMessage(message); + return message; +} + +function formatDiagnosticId(value: string | undefined | null) { + return value ? `present(length=${value.length})` : "missing"; +} + +function formatWhatsAppMessageShape(message: WhatsAppQaDriverObservedMessage, index: number) { + return [ + `#${index + 1}`, + `observedAt=${message.observedAt}`, + `fromPhone=${message.fromPhoneE164 ? "present" : "missing"}`, + `kind=${message.kind}`, + `textLength=${message.text.length}`, + `messageId=${formatDiagnosticId(message.messageId)}`, + `quoted=${message.quoted ? "present" : "missing"}`, + `quotedMessageId=${formatDiagnosticId(message.quoted?.messageId)}`, + ].join(" "); +} + +function formatWhatsAppScenarioWaitDiagnostics( + context: WhatsAppQaObservedMessagesContext, + params: { + diagnosticChecks?: Array<{ + label: string; + match: (message: WhatsAppQaDriverObservedMessage) => boolean; + }>; + observedAfter?: Date; + }, +) { + const lowerBoundMs = params.observedAfter?.getTime(); + const messages = context.driver.getObservedMessages().filter((message) => { + if (lowerBoundMs === undefined) { + return true; + } + return new Date(message.observedAt).getTime() >= lowerBoundMs; + }); + if (messages.length === 0) { + return "observed 0 WhatsApp driver message(s) after wait lower bound"; + } + const formatted = messages.slice(-5).map((message, index) => { + const checks = (params.diagnosticChecks ?? []).map((check) => { + try { + const matched = check.match(message); + return `${check.label}=${matched ? "yes" : "no"}`; + } catch { + return `${check.label}=no`; + } + }); + return [ + formatWhatsAppMessageShape(message, index), + `fromExpectedSut=${message.fromPhoneE164 === context.sutPhoneE164 ? "yes" : "no"}`, + ...checks, + ].join(" "); + }); + return `observed ${messages.length} WhatsApp driver message(s) after wait lower bound: ${formatted.join("; ")}`; +} + +function hasWhatsAppBatchExpectations(run: WhatsAppQaMessageScenarioRun) { + return ( + run.expectedSutMessageCount !== undefined || + run.expectedSutMessageCountRange !== undefined || + (run.expectedJoinedSutTextIncludes?.length ?? 0) > 0 + ); +} + +function isWhatsAppScenarioSutMessage( + message: WhatsAppQaDriverObservedMessage, + params: { + observedAfter: Date; + sutPhoneE164: string; + target: string; + targetKind: "dm" | "group"; + }, +) { + if (new Date(message.observedAt).getTime() < params.observedAfter.getTime()) { + return false; + } + if (params.targetKind === "group") { + return ( + message.fromJid === params.target && + (!message.fromPhoneE164 || message.fromPhoneE164 === params.sutPhoneE164) + ); + } + return message.fromPhoneE164 === params.sutPhoneE164; +} + +async function assertWhatsAppScenarioMessageBatch(params: { + alreadyRecordedMessageIds: Set; + context: WhatsAppQaMessageScenarioContext; + observedAfter: Date; + run: WhatsAppQaMessageScenarioRun; +}) { + if (!hasWhatsAppBatchExpectations(params.run)) { + return undefined; + } + await new Promise((resolve) => { + setTimeout(resolve, params.run.settleMs ?? 4_000); + }); + const messages = params.context.driver.getObservedMessages().filter((message) => + isWhatsAppScenarioSutMessage(message, { + observedAfter: params.observedAfter, + sutPhoneE164: params.context.sutPhoneE164, + target: params.context.target, + targetKind: params.run.target, + }), + ); + if ( + params.run.expectedSutMessageCount !== undefined && + messages.length !== params.run.expectedSutMessageCount + ) { + throw new Error( + `expected ${params.run.expectedSutMessageCount} SUT message(s), observed ${ + messages.length + }: ${formatWhatsAppBatchMessageDiagnostics(messages)}`, + ); + } + if (params.run.expectedSutMessageCountRange !== undefined) { + const [min, max] = params.run.expectedSutMessageCountRange; + if (messages.length < min || messages.length > max) { + throw new Error( + `expected ${min}-${max} SUT message(s), observed ${ + messages.length + }: ${formatWhatsAppBatchMessageDiagnostics(messages)}`, + ); + } + } + const joinedText = messages.map((message) => message.text).join("\n"); + for (const expected of params.run.expectedJoinedSutTextIncludes ?? []) { + if (!joinedText.includes(expected)) { + throw new Error(`expected joined WhatsApp SUT text to include ${expected}`); + } + } + for (const message of messages) { + if (!message.messageId || params.alreadyRecordedMessageIds.has(message.messageId)) { + continue; + } + params.context.recordObservedMessage(message); + params.alreadyRecordedMessageIds.add(message.messageId); + } + return `observed ${messages.length} SUT message(s) after settle`; +} + +function formatWhatsAppBatchMessageDiagnostics(messages: WhatsAppQaDriverObservedMessage[]) { + if (messages.length === 0) { + return "no matching SUT message shapes observed"; + } + return messages.slice(-5).map(formatWhatsAppMessageShape).join("; "); +} + +function findUnexpectedWhatsAppNoReplyMessage(params: { + groupJid?: string; + messages: WhatsAppQaDriverObservedMessage[]; + observedAfter: Date; + sutPhoneE164: string; + target: "dm" | "group"; +}): WhatsAppQaDriverObservedMessage | undefined { + const observedAfterMs = params.observedAfter.getTime(); + return params.messages.find((message) => { + if (new Date(message.observedAt).getTime() < observedAfterMs) { + return false; + } + if (params.target === "group") { + return message.fromJid === params.groupJid; + } + return message.fromPhoneE164 === params.sutPhoneE164; + }); +} + function isTransientWhatsAppQaDriverError(error: unknown) { const message = formatErrorMessage(error); return ( /\bConnection Closed\b/iu.test(message) || /\bconflict\b/iu.test(message) || - /\bsession conflict\b/iu.test(message) || - /\btimed out waiting for WhatsApp QA driver message\b/iu.test(message) + /\bpending notifications\b/iu.test(message) || + /\bsession conflict\b/iu.test(message) ); } @@ -622,7 +2135,7 @@ async function restartWhatsAppQaDriverSession(params: { current: WhatsAppQaDriverSession; }) { await params.current.close().catch(() => {}); - return await startWhatsAppQaDriverSession({ authDir: params.authDir }); + return await startWhatsAppQaDriverSessionWithRetry({ authDir: params.authDir }); } async function startWhatsAppQaDriverSessionWithRetry(params: { authDir: string }) { @@ -631,7 +2144,10 @@ async function startWhatsAppQaDriverSessionWithRetry(params: { authDir: string } (_, index) => index + 1, )) { try { - return await startWhatsAppQaDriverSession({ authDir: params.authDir }); + return await startWhatsAppQaDriverSession({ + authDir: params.authDir, + waitForPendingNotifications: true, + }); } catch (error) { if ( attempt >= WHATSAPP_QA_TRANSIENT_DRIVER_ATTEMPTS || @@ -808,17 +2324,80 @@ function matchesWhatsAppApprovalPendingText(params: { function matchesWhatsAppApprovalResolvedText(params: { approvalId: string; approvalKind: WhatsAppQaApprovalKind; + decision?: WhatsAppQaApprovalDecision; text: string; }) { + const decision = params.decision ?? "allow-once"; + const decisionText = + params.approvalKind === "exec" + ? decision + : decision === "allow-once" + ? "allowed once" + : "denied"; const heading = - params.approvalKind === "exec" ? "Exec approval allow-once" : "Plugin approval allowed once"; + params.approvalKind === "exec" + ? `Exec approval ${decisionText}` + : `Plugin approval ${decisionText}`; return params.text.includes(params.approvalId) && params.text.includes(heading); } +function formatWhatsAppApprovalWaitDiagnostics(params: { + approvalId: string; + approvalKind: WhatsAppQaApprovalKind; + decision?: WhatsAppQaApprovalDecision; + driver: WhatsAppQaDriverSession; + observedAfter?: Date; + state: "pending" | "resolved"; + sutPhoneE164: string; + token: string; +}) { + const lowerBoundMs = params.observedAfter?.getTime(); + const messages = params.driver.getObservedMessages().filter((message) => { + if (lowerBoundMs === undefined) { + return true; + } + return new Date(message.observedAt).getTime() >= lowerBoundMs; + }); + if (messages.length === 0) { + return `observed 0 WhatsApp driver message(s) after ${params.state} approval wait lower bound`; + } + const formatted = messages.slice(-5).map((message, index) => { + const fromExpectedSender = + !message.fromPhoneE164 || message.fromPhoneE164 === params.sutPhoneE164; + const approvalTextMatches = + params.state === "pending" + ? matchesWhatsAppApprovalPendingText({ + approvalId: params.approvalId, + approvalKind: params.approvalKind, + text: message.text, + token: params.token, + }) + : matchesWhatsAppApprovalResolvedText({ + approvalId: params.approvalId, + approvalKind: params.approvalKind, + decision: params.decision, + text: message.text, + }); + return [ + `#${index + 1}`, + `observedAt=${message.observedAt}`, + `fromExpectedSut=${fromExpectedSender ? "yes" : "no"}`, + `fromPhone=${message.fromPhoneE164 ? "present" : "missing"}`, + `kind=${message.kind}`, + `textLength=${message.text.length}`, + `approvalText=${approvalTextMatches ? "yes" : "no"}`, + `messageId=${formatDiagnosticId(message.messageId)}`, + ].join(" "); + }); + return `observed ${messages.length} WhatsApp driver message(s) after ${params.state} approval wait lower bound: ${formatted.join("; ")}`; +} + async function waitForWhatsAppApprovalMessage(params: { approvalId: string; approvalKind: WhatsAppQaApprovalKind; + decision?: WhatsAppQaApprovalDecision; driver: WhatsAppQaDriverSession; + observedAfter?: Date; observedMessages: WhatsAppObservedMessage[]; scenario: WhatsAppQaScenarioDefinition; state: "pending" | "resolved"; @@ -826,28 +2405,41 @@ async function waitForWhatsAppApprovalMessage(params: { timeoutMs: number; token: string; }) { - const reply = await params.driver.waitForMessage({ - timeoutMs: params.timeoutMs, - match: (message) => { - const fromExpectedSender = - !message.fromPhoneE164 || message.fromPhoneE164 === params.sutPhoneE164; - return ( - fromExpectedSender && - (params.state === "pending" - ? matchesWhatsAppApprovalPendingText({ - approvalId: params.approvalId, - approvalKind: params.approvalKind, - text: message.text, - token: params.token, - }) - : matchesWhatsAppApprovalResolvedText({ - approvalId: params.approvalId, - approvalKind: params.approvalKind, - text: message.text, - })) + let reply: WhatsAppQaDriverObservedMessage; + try { + reply = await params.driver.waitForMessage({ + observedAfter: params.observedAfter, + timeoutMs: params.timeoutMs, + match: (message) => { + const fromExpectedSender = + !message.fromPhoneE164 || message.fromPhoneE164 === params.sutPhoneE164; + return ( + fromExpectedSender && + (params.state === "pending" + ? matchesWhatsAppApprovalPendingText({ + approvalId: params.approvalId, + approvalKind: params.approvalKind, + text: message.text, + token: params.token, + }) + : matchesWhatsAppApprovalResolvedText({ + approvalId: params.approvalId, + approvalKind: params.approvalKind, + decision: params.decision, + text: message.text, + })) + ); + }, + }); + } catch (error) { + if (/\btimed out waiting for WhatsApp QA driver message\b/iu.test(formatErrorMessage(error))) { + throw new Error( + `${formatErrorMessage(error)}; ${formatWhatsAppApprovalWaitDiagnostics(params)}`, + { cause: error }, ); - }, - }); + } + throw error; + } const observed: WhatsAppObservedMessage = { ...reply, approvalState: params.state, @@ -881,10 +2473,12 @@ async function runWhatsAppApprovalScenario(params: { run: params.run, sutAccountId: params.sutAccountId, }); - await waitForWhatsAppApprovalMessage({ + const pending = await waitForWhatsAppApprovalMessage({ approvalId, approvalKind: params.run.approvalKind, + decision: params.run.decision, driver: params.driver, + observedAfter: requestStartedAt, observedMessages: params.observedMessages, scenario: params.scenario, state: "pending", @@ -895,7 +2489,9 @@ async function runWhatsAppApprovalScenario(params: { const resolvedPromise = waitForWhatsAppApprovalMessage({ approvalId, approvalKind: params.run.approvalKind, + decision: params.run.decision, driver: params.driver, + observedAfter: requestStartedAt, observedMessages: params.observedMessages, scenario: params.scenario, state: "resolved", @@ -904,12 +2500,21 @@ async function runWhatsAppApprovalScenario(params: { token: params.run.token, }); try { - await resolveApprovalDecision({ - approvalId, - decision: params.run.decision, - gateway: params.gateway, - kind: params.run.approvalKind, - }); + if (params.run.decisionMode === "reaction") { + if (!pending.fromJid || !pending.messageId) { + throw new Error("WhatsApp approval prompt did not expose message coordinates."); + } + await params.driver.sendReaction(pending.fromJid, pending.messageId, "👍", { + fromMe: false, + }); + } else { + await resolveApprovalDecision({ + approvalId, + decision: params.run.decision, + gateway: params.gateway, + kind: params.run.approvalKind, + }); + } assertApprovalDecisionResult({ decision: params.run.decision, result: await waitForApprovalDecision({ @@ -952,22 +2557,36 @@ async function runWhatsAppScenario(params: { if (scenarioRun.kind !== "approval" && scenarioRun.target === "group" && !params.groupJid) { throw new Error(`WhatsApp scenario ${params.scenario.id} requires groupJid.`); } - const target = - scenarioRun.kind !== "approval" && scenarioRun.target === "group" - ? params.groupJid! - : params.sutPhoneE164; + const targets = + scenarioRun.kind !== "approval" + ? resolveWhatsAppQaMessageTargets({ + driverPhoneE164: params.driverPhoneE164, + groupJid: params.groupJid, + scenarioTarget: scenarioRun.target, + sutPhoneE164: params.sutPhoneE164, + }) + : undefined; + const target = targets?.driverTarget ?? params.sutPhoneE164; const allowFrom = - scenarioRun.kind === "approval" || scenarioRun.configMode === "allowlist" + scenarioRun.kind === "approval" ? [params.driverPhoneE164] - : ["+15550000000"]; + : scenarioRun.configMode === "open" + ? ["*"] + : scenarioRun.configMode === "pairing" + ? ["+15550000000"] + : [params.driverPhoneE164]; const dmPolicy = - scenarioRun.kind === "approval" || scenarioRun.configMode === "allowlist" + scenarioRun.kind === "approval" ? "allowlist" - : "pairing"; + : scenarioRun.configMode === "open" || scenarioRun.configMode === "disabled" + ? scenarioRun.configMode + : scenarioRun.configMode === "allowlist" + ? "allowlist" + : "pairing"; const gatewayHarness = await startQaLiveLaneGateway({ repoRoot: params.repoRoot, transport: { - requiredPluginIds: [], + requiredPluginIds: params.scenario.requiredPluginIds ?? [], createGatewayConfig: () => ({}), }, transportBaseUrl: "http://127.0.0.1:0", @@ -1024,9 +2643,22 @@ async function runWhatsAppScenario(params: { }, }; } - if (scenarioRun.quietInput) { + if (scenarioRun.quietInput !== undefined) { const quietStartedAt = new Date(); - await params.driver.sendText(target, scenarioRun.quietInput); + const quietSendMode = scenarioRun.quietSendMode ?? scenarioRun.sendMode; + if (quietSendMode?.kind === "media") { + await params.driver.sendMedia( + target, + scenarioRun.quietInput, + quietSendMode.mediaBuffer, + quietSendMode.mediaType, + { + fileName: quietSendMode.fileName, + }, + ); + } else { + await params.driver.sendText(target, scenarioRun.quietInput); + } await new Promise((resolve) => { setTimeout(resolve, scenarioRun.quietWindowMs ?? 5_000); }); @@ -1046,11 +2678,58 @@ async function runWhatsAppScenario(params: { } } const requestStartedAt = new Date(); - await params.driver.sendText(target, scenarioRun.input); + const sent = + scenarioRun.sendMode?.kind === "media" + ? await params.driver.sendMedia( + target, + scenarioRun.input, + scenarioRun.sendMode.mediaBuffer, + scenarioRun.sendMode.mediaType, + { + fileName: scenarioRun.sendMode.fileName, + }, + ) + : await params.driver.sendText(target, scenarioRun.input); + const scenarioContext: WhatsAppQaMessageScenarioContext = { + driver: params.driver, + driverPhoneE164: params.driverPhoneE164, + gateway: gatewayHarness.gateway, + gatewayTarget: targets?.gatewayTarget ?? params.driverPhoneE164, + gatewayWorkspaceDir: gatewayHarness.gateway.workspaceDir, + recordObservedMessage: (message) => { + params.observedMessages.push({ + ...message, + matchedScenario: true, + scenarioId: params.scenario.id, + scenarioTitle: params.scenario.title, + }); + }, + requestStartedAt, + scenarioId: params.scenario.id, + scenarioTitle: params.scenario.title, + sent, + sutAccountId: params.sutAccountId, + sutPhoneE164: params.sutPhoneE164, + target, + waitForReady: async () => { + await waitForWhatsAppChannelStable(gatewayHarness.gateway, params.sutAccountId); + }, + }; + const afterSendDetails = await scenarioRun.afterSend?.(scenarioContext); if (!scenarioRun.expectReply) { await new Promise((resolve) => { setTimeout(resolve, params.scenario.timeoutMs); }); + const unexpectedReply = findUnexpectedWhatsAppNoReplyMessage({ + groupJid: params.groupJid, + messages: params.driver.getObservedMessages(), + observedAfter: requestStartedAt, + sutPhoneE164: params.sutPhoneE164, + target: scenarioRun.target, + }); + if (unexpectedReply) { + throw new Error("unexpected WhatsApp reply observed in quiet scenario"); + } return { id: params.scenario.id, title: params.scenario.title, @@ -1059,6 +2738,7 @@ async function runWhatsAppScenario(params: { }; } const reply = await params.driver.waitForMessage({ + observedAfter: requestStartedAt, timeoutMs: params.scenario.timeoutMs, match: (message) => (scenarioRun.target === "group" @@ -1072,14 +2752,24 @@ async function runWhatsAppScenario(params: { scenarioId: params.scenario.id, scenarioTitle: params.scenario.title, }; + scenarioRun.verify?.(reply, scenarioContext); params.observedMessages.push(observed); + const afterReplyDetails = await scenarioRun.afterReply?.(reply, scenarioContext); + const batchDetails = await assertWhatsAppScenarioMessageBatch({ + alreadyRecordedMessageIds: new Set(observed.messageId ? [observed.messageId] : []), + context: scenarioContext, + observedAfter: requestStartedAt, + run: scenarioRun, + }); const responseObservedAt = new Date(reply.observedAt); const rttMs = responseObservedAt.getTime() - requestStartedAt.getTime(); return { id: params.scenario.id, title: params.scenario.title, status: "pass" as const, - details: `reply matched in ${rttMs}ms`, + details: [`reply matched in ${rttMs}ms`, afterSendDetails, afterReplyDetails, batchDetails] + .filter(Boolean) + .join("; "), rttMs, requestStartedAt: requestStartedAt.toISOString(), responseObservedAt: responseObservedAt.toISOString(), @@ -1092,9 +2782,7 @@ async function runWhatsAppScenario(params: { }; } catch (error) { preservedGatewayDebug = true; - await gatewayHarness.gateway - .stop({ preserveToDir: params.gatewayDebugDirPath }) - .catch(() => {}); + await gatewayHarness.stop({ preserveToDir: params.gatewayDebugDirPath }).catch(() => {}); throw error; } finally { if (!preservedGatewayDebug) { @@ -1111,15 +2799,67 @@ function toObservedWhatsAppArtifacts(params: { return params.messages.map((message) => ({ approvalState: message.approvalState, fromPhoneE164: params.redactMetadata ? undefined : message.fromPhoneE164, + hasMedia: message.hasMedia, + kind: message.kind, matchedScenario: message.matchedScenario, + mediaFileName: params.redactMetadata ? undefined : message.mediaFileName, + mediaType: message.mediaType, messageId: params.redactMetadata ? undefined : message.messageId, observedAt: message.observedAt, + poll: params.includeContent ? message.poll : undefined, + quoted: formatObservedWhatsAppQuotedArtifact(message.quoted, { + includeContent: params.includeContent, + redactMetadata: params.redactMetadata, + }), + reaction: formatObservedWhatsAppReactionArtifact(message.reaction, { + includeContent: params.includeContent, + redactMetadata: params.redactMetadata, + }), scenarioId: message.scenarioId, scenarioTitle: message.scenarioTitle, text: params.includeContent ? message.text : undefined, })); } +function formatObservedWhatsAppReactionArtifact( + reaction: WhatsAppQaDriverObservedMessage["reaction"], + params: { includeContent: boolean; redactMetadata: boolean }, +): WhatsAppObservedReactionArtifact | undefined { + if (!reaction) { + return undefined; + } + const artifact: WhatsAppObservedReactionArtifact = {}; + if (params.includeContent) { + artifact.emoji = reaction.emoji; + } + if (reaction.fromMe !== undefined) { + artifact.fromMe = reaction.fromMe; + } + if (!params.redactMetadata) { + if (reaction.messageId !== undefined) { + artifact.messageId = reaction.messageId; + } + if (reaction.participant !== undefined) { + artifact.participant = reaction.participant; + } + } + return artifact; +} + +function formatObservedWhatsAppQuotedArtifact( + quoted: WhatsAppQaDriverObservedMessage["quoted"], + params: { includeContent: boolean; redactMetadata: boolean }, +) { + if (!quoted) { + return undefined; + } + return { + messageId: params.redactMetadata ? undefined : quoted.messageId, + participant: params.redactMetadata ? undefined : quoted.participant, + text: params.includeContent ? quoted.text : undefined, + }; +} + function renderWhatsAppQaMarkdown(params: { cleanupIssues: string[]; credentialFingerprint?: string; @@ -1165,6 +2905,39 @@ function renderWhatsAppQaMarkdown(params: { return lines.join("\n"); } +function redactWhatsAppQaScenarioResults( + scenarios: readonly WhatsAppQaScenarioResult[], +): WhatsAppQaScenarioResult[] { + return scenarios.map((scenario) => ({ + ...scenario, + details: redactWhatsAppQaScenarioDetails(scenario.details), + })); +} + +const SAFE_WHATSAPP_DRIVER_DIAGNOSTICS_PATTERN = + /observed \d+ WhatsApp driver message\(s\) after (?:(?:pending|resolved) approval )?wait lower bound(?:: [-A-Za-z0-9_#:=()., +;/]+)?/u; + +function isRedactionSafeWhatsAppScenarioDetailSegment(segment: string) { + return ( + /^no reply$/u.test(segment) || + /^reply matched in \d+ms$/u.test(segment) || + /^observed \d+ SUT message\(s\) after settle$/u.test(segment) + ); +} + +function redactWhatsAppQaScenarioDetails(details: string) { + const normalized = details.trim(); + const safeDriverDiagnostics = normalized.match(SAFE_WHATSAPP_DRIVER_DIAGNOSTICS_PATTERN); + if (safeDriverDiagnostics) { + return safeDriverDiagnostics[0]; + } + const safeSegments = normalized + .split(";") + .map((segment) => segment.trim()) + .filter(isRedactionSafeWhatsAppScenarioDetailSegment); + return safeSegments.length > 0 ? safeSegments.join("; ") : redactQaLiveLaneDetails(); +} + function createMissingGroupJidScenarioResult(params: { explicitScenarioSelection: boolean; scenario: WhatsAppQaScenarioDefinition; @@ -1200,6 +2973,31 @@ function appendPreScenarioFailureResults(params: { } } +function formatWhatsAppScenarioProgressLine(params: { + details?: string; + index: number; + scenario: WhatsAppQaScenarioDefinition; + status: "fail" | "pass" | "skip" | "start"; + total: number; +}) { + const prefix = `[whatsapp-qa] [${params.index}/${params.total}] ${params.status}`; + const detailSuffix = params.details ? ` - ${params.details}` : ""; + return `${prefix} ${params.scenario.id}: ${params.scenario.title}${detailSuffix}`; +} + +function formatWhatsAppScenarioProgressDetails(params: { + details: string; + redactMetadata: boolean; +}) { + return params.redactMetadata ? redactWhatsAppQaScenarioDetails(params.details) : params.details; +} + +function logWhatsAppScenarioProgress( + params: Parameters[0], +) { + process.stderr.write(`${formatWhatsAppScenarioProgressLine(params)}\n`); +} + export async function runWhatsAppQaLive(params: { alternateModel?: string; credentialRole?: string; @@ -1224,7 +3022,7 @@ export async function runWhatsAppQaLive(params: { const primaryModel = params.primaryModel?.trim() || defaultQaModelForMode(providerMode); const alternateModel = params.alternateModel?.trim() || defaultQaModelForMode(providerMode, true); const sutAccountId = params.sutAccountId?.trim() || "sut"; - const scenarios = findScenarios(params.scenarioIds); + const scenarios = findScenarios(params.scenarioIds, providerMode); const explicitScenarioSelection = (params.scenarioIds?.length ?? 0) > 0; const requestedCredentialSource = inferWhatsAppCredentialSource(params.credentialSource); const requestedCredentialRole = inferWhatsAppCredentialRole(params.credentialRole); @@ -1240,7 +3038,7 @@ export async function runWhatsAppQaLive(params: { let leaseHeartbeat: WhatsAppCredentialHeartbeat | undefined; let runtimeEnv: WhatsAppQaRuntimeEnv | undefined; let tempAuthRoot: string | undefined; - let driver: WhatsAppQaDriverSession | undefined; + let closeDriverSession: (() => Promise) | undefined; try { credentialLease = await acquireQaCredentialLease({ @@ -1271,17 +3069,33 @@ export async function runWhatsAppQaLive(params: { }), ]); let activeDriver = await startWhatsAppQaDriverSessionWithRetry({ authDir: driverAuthDir }); - driver = activeDriver; + closeDriverSession = () => activeDriver.close(); - for (const scenario of scenarios) { + for (const [scenarioIndex, scenario] of scenarios.entries()) { + const progressIndex = scenarioIndex + 1; + logWhatsAppScenarioProgress({ + index: progressIndex, + scenario, + status: "start", + total: scenarios.length, + }); assertLeaseHealthy(); if (scenario.requiresGroupJid && !runtimeEnv.groupJid) { - scenarioResults.push( - createMissingGroupJidScenarioResult({ - explicitScenarioSelection, - scenario, + const result = createMissingGroupJidScenarioResult({ + explicitScenarioSelection, + scenario, + }); + scenarioResults.push(result); + logWhatsAppScenarioProgress({ + details: formatWhatsAppScenarioProgressDetails({ + details: result.details, + redactMetadata: redactPublicMetadata, }), - ); + index: progressIndex, + scenario, + status: result.status, + total: scenarios.length, + }); continue; } let driverAttempt = 1; @@ -1303,14 +3117,24 @@ export async function runWhatsAppQaLive(params: { sutAuthDir, sutPhoneE164: runtimeEnv.sutPhoneE164, }); - scenarioResults.push( + const recordedResult = driverAttempt > 1 ? { ...result, details: `${result.details}; driver reconnected ${driverAttempt - 1}x`, } - : result, - ); + : result; + scenarioResults.push(recordedResult); + logWhatsAppScenarioProgress({ + details: formatWhatsAppScenarioProgressDetails({ + details: recordedResult.details, + redactMetadata: redactPublicMetadata, + }), + index: progressIndex, + scenario, + status: recordedResult.status, + total: scenarios.length, + }); break; } catch (error) { if ( @@ -1321,21 +3145,15 @@ export async function runWhatsAppQaLive(params: { await new Promise((resolve) => { setTimeout(resolve, WHATSAPP_QA_DRIVER_RECONNECT_DELAY_MS); }); - try { - activeDriver = await restartWhatsAppQaDriverSession({ - authDir: driverAuthDir, - current: activeDriver, - }); - driver = activeDriver; - } catch (restartError) { - if (!isTransientWhatsAppQaDriverError(restartError)) { - throw restartError; - } - } + activeDriver = await restartWhatsAppQaDriverSession({ + authDir: driverAuthDir, + current: activeDriver, + }); + closeDriverSession = () => activeDriver.close(); continue; } preservedGatewayDebugArtifacts = true; - scenarioResults.push({ + const result: WhatsAppQaScenarioResult = { id: scenario.id, title: scenario.title, status: "fail", @@ -1343,6 +3161,17 @@ export async function runWhatsAppQaLive(params: { driverAttempt > 1 ? `${formatErrorMessage(error)}; driver reconnected ${driverAttempt - 1}x` : formatErrorMessage(error), + }; + scenarioResults.push(result); + logWhatsAppScenarioProgress({ + details: formatWhatsAppScenarioProgressDetails({ + details: result.details, + redactMetadata: redactPublicMetadata, + }), + index: progressIndex, + scenario, + status: "fail", + total: scenarios.length, }); break; } @@ -1369,9 +3198,9 @@ export async function runWhatsAppQaLive(params: { scenarios, }); } finally { - if (driver) { + if (closeDriverSession) { try { - await driver.close(); + await closeDriverSession(); } catch (error) { appendLiveLaneIssue(cleanupIssues, "driver session stop failed", error); } @@ -1405,6 +3234,12 @@ export async function runWhatsAppQaLive(params: { const failed = scenarioResults.filter((entry) => entry.status === "fail").length; const skipped = scenarioResults.filter((entry) => entry.status === "skip").length; const credentialFingerprint = fingerprintQaCredentialId(credentialLease?.credentialId); + const publishedCleanupIssues = redactPublicMetadata + ? redactQaLiveLaneIssues(cleanupIssues) + : cleanupIssues; + const publishedScenarioResults = redactPublicMetadata + ? redactWhatsAppQaScenarioResults(scenarioResults) + : scenarioResults; const summary: WhatsAppQaSummary = { credentials: credentialLease ? { @@ -1426,14 +3261,14 @@ export async function runWhatsAppQaLive(params: { : (runtimeEnv?.sutPhoneE164 ?? ""), startedAt, finishedAt, - cleanupIssues, + cleanupIssues: publishedCleanupIssues, counts: { total: scenarioResults.length, passed, failed, skipped, }, - scenarios: scenarioResults, + scenarios: publishedScenarioResults, }; await fs.writeFile( observedMessagesPath, @@ -1451,13 +3286,13 @@ export async function runWhatsAppQaLive(params: { await fs.writeFile( reportPath, `${renderWhatsAppQaMarkdown({ - cleanupIssues, + cleanupIssues: publishedCleanupIssues, credentialFingerprint, credentialSource: credentialLease?.source ?? requestedCredentialSource, finishedAt, gatewayDebugDirPath: preservedGatewayDebugArtifacts ? gatewayDebugDirPath : undefined, redactMetadata: redactPublicMetadata, - scenarios: scenarioResults, + scenarios: publishedScenarioResults, startedAt, sutPhoneE164: runtimeEnv?.sutPhoneE164, })}\n`, @@ -1476,13 +3311,25 @@ export const testing = { assertSafeArchiveEntries, appendPreScenarioFailureResults, buildWhatsAppQaConfig, + callWhatsAppGatewayMessageAction, + callWhatsAppGatewayPoll, + callWhatsAppGatewaySend, createMissingGroupJidScenarioResult, findScenarios, + findUnexpectedWhatsAppNoReplyMessage, + formatWhatsAppApprovalWaitDiagnostics, + formatWhatsAppBatchMessageDiagnostics, + formatWhatsAppScenarioProgressDetails, + formatWhatsAppScenarioProgressLine, fingerprintWhatsAppCredentialId: fingerprintQaCredentialId, + formatWhatsAppScenarioWaitDiagnostics, isTransientWhatsAppQaDriverError, matchesWhatsAppApprovalResolvedText, parseWhatsAppQaCredentialPayload, renderWhatsAppQaMarkdown, + runWhatsAppStructuredInboundChecks, + redactWhatsAppQaScenarioResults, + resolveWhatsAppQaMessageTargets, resolveWhatsAppQaRuntimeEnv, resolveWhatsAppMetadataRedaction, toObservedWhatsAppArtifacts, diff --git a/extensions/qa-lab/src/providers/mock-openai/server.test.ts b/extensions/qa-lab/src/providers/mock-openai/server.test.ts index 39e4fbb42e1e..f21e07b59ccc 100644 --- a/extensions/qa-lab/src/providers/mock-openai/server.test.ts +++ b/extensions/qa-lab/src/providers/mock-openai/server.test.ts @@ -293,6 +293,26 @@ describe("qa mock openai server", () => { expect(telegramLongBody).toContain("TELEGRAM-LONG-FINAL-END"); expect(telegramLongBody.length).toBeGreaterThan(4_500); + const whatsappLongResponse = await fetch(`${server.baseUrl}/v1/responses`, { + method: "POST", + headers: { + "content-type": "application/json", + }, + body: JSON.stringify({ + stream: true, + input: [ + makeUserInput("WhatsApp long final QA check. Use the scripted long final response."), + ], + }), + }); + expect(whatsappLongResponse.status).toBe(200); + const whatsappLongBody = await whatsappLongResponse.text(); + expect(whatsappLongBody).toContain('"type":"response.output_text.delta"'); + expect(whatsappLongBody).toContain('"phase":"final_answer"'); + expect(whatsappLongBody).toContain("WHATSAPP-LONG-FINAL-BEGIN"); + expect(whatsappLongBody).toContain("WHATSAPP-LONG-FINAL-END"); + expect(whatsappLongBody.length).toBeGreaterThan(6_000); + const telegramThreeChunkLongResponse = await fetch(`${server.baseUrl}/v1/responses`, { method: "POST", headers: { @@ -2856,6 +2876,172 @@ describe("qa mock openai server", () => { expect(outputText(await response.json())).toBe("QA_CANARY_TEST"); }); + it("uses WhatsApp location markers only for the matching coordinate body", async () => { + const server = await startMockServer(); + const setupInput = makeUserInput( + "When a later WhatsApp location message shows 37.774900, -122.419400, " + + "reply with only this WhatsApp location marker: QA_WHATSAPP_LOCATION_OK. " + + "Reply with only this exact marker: QA_INITIAL_OK", + ); + + const setupResponse = await postResponses(server, { + stream: false, + input: [setupInput], + }); + + const response = await postResponses(server, { + stream: false, + input: [setupInput, makeUserInput("📍 37.774900, -122.419400")], + }); + + expect(setupResponse.status).toBe(200); + expect(outputText(await setupResponse.json())).toBe("QA_INITIAL_OK"); + expect(response.status).toBe(200); + expect(outputText(await response.json())).toBe("QA_WHATSAPP_LOCATION_OK"); + }); + + it("uses WhatsApp contact and sticker markers only for matching structured bodies", async () => { + const server = await startMockServer(); + const setupInput = makeUserInput( + "When a later WhatsApp contact message appears, " + + "reply with only this WhatsApp contact marker: QA_WHATSAPP_CONTACT_OK. " + + "When a later WhatsApp sticker message appears, " + + "reply with only this WhatsApp sticker marker: QA_WHATSAPP_STICKER_OK. " + + "Reply with only this exact marker: QA_STRUCTURED_INITIAL_OK", + ); + + const setupResponse = await postResponses(server, { + stream: false, + input: [setupInput], + }); + const contactResponse = await postResponses(server, { + stream: false, + input: [setupInput, makeUserInput("")], + }); + const stickerResponse = await postResponses(server, { + stream: false, + input: [setupInput, makeUserInput("")], + }); + + expect(setupResponse.status).toBe(200); + expect(outputText(await setupResponse.json())).toBe("QA_STRUCTURED_INITIAL_OK"); + expect(contactResponse.status).toBe(200); + expect(outputText(await contactResponse.json())).toBe("QA_WHATSAPP_CONTACT_OK"); + expect(stickerResponse.status).toBe(200); + expect(outputText(await stickerResponse.json())).toBe("QA_WHATSAPP_STICKER_OK"); + }); + + it("uses WhatsApp structured markers for channel-prefixed message bodies", async () => { + const server = await startMockServer(); + const setupInput = makeUserInput( + "When a later WhatsApp location message shows 37.774900, -122.419400, " + + "reply with only this WhatsApp location marker: QA_WHATSAPP_LOCATION_OK. " + + "When a later WhatsApp contact message appears, " + + "reply with only this WhatsApp contact marker: QA_WHATSAPP_CONTACT_OK. " + + "When a later WhatsApp sticker message appears, " + + "reply with only this WhatsApp sticker marker: QA_WHATSAPP_STICKER_OK. " + + "Reply with only this exact marker: QA_STRUCTURED_INITIAL_OK", + ); + const previousExactMarkerInput = makeUserInput( + "Reply with only this previous unrelated exact marker: QA_WHATSAPP_PREVIOUS_OK", + ); + + const locationResponse = await postResponses(server, { + stream: false, + input: [ + setupInput, + previousExactMarkerInput, + makeUserInput( + [ + "Conversation info (untrusted metadata):", + "```json", + '{"inbound_event_kind":"user_request"}', + "```", + "", + "📍 37.774900, -122.419400", + ].join("\n"), + ), + ], + }); + const contactResponse = await postResponses(server, { + stream: false, + input: [ + setupInput, + previousExactMarkerInput, + makeUserInput( + ["Sender (untrusted metadata):", "```json", '{"name":"QA"}', "```", "", ""].join( + "\n", + ), + ), + ], + }); + const stickerResponse = await postResponses(server, { + stream: false, + input: [ + setupInput, + previousExactMarkerInput, + makeUserInput( + [ + "Conversation info (untrusted metadata):", + "```json", + '{"inbound_event_kind":"user_request"}', + "```", + "", + "", + ].join("\n"), + ), + ], + }); + + expect(locationResponse.status).toBe(200); + expect(outputText(await locationResponse.json())).toBe("QA_WHATSAPP_LOCATION_OK"); + expect(contactResponse.status).toBe(200); + expect(outputText(await contactResponse.json())).toBe("QA_WHATSAPP_CONTACT_OK"); + expect(stickerResponse.status).toBe(200); + expect(outputText(await stickerResponse.json())).toBe("QA_WHATSAPP_STICKER_OK"); + }); + + it("streams WhatsApp location markers for the matching coordinate body", async () => { + const server = await startMockServer(); + + const body = await expectResponsesText(server, { + stream: true, + input: [ + makeUserInput( + "When a later WhatsApp location message shows 37.774900, -122.419400, " + + "reply with only this WhatsApp location marker: QA_WHATSAPP_LOCATION_STREAM_OK. " + + "Reply with only this exact marker: QA_INITIAL_STREAM_OK", + ), + makeUserInput("📍 37.774900, -122.419400"), + ], + }); + + expect(body).toContain("QA_WHATSAPP_LOCATION_STREAM_OK"); + expect(body).not.toContain("QA_INITIAL_STREAM_OK"); + }); + + it("streams WhatsApp structured markers ahead of previous exact markers", async () => { + const server = await startMockServer(); + + const body = await expectResponsesText(server, { + stream: true, + input: [ + makeUserInput( + "When a later WhatsApp location message shows 37.774900, -122.419400, " + + "reply with only this WhatsApp location marker: QA_WHATSAPP_LOCATION_STREAM_OK. " + + "Reply with only this exact marker: QA_INITIAL_STREAM_OK", + ), + makeUserInput( + "Reply with only this previous unrelated exact marker: QA_WHATSAPP_PREVIOUS_STREAM_OK", + ), + makeUserInput("📍 37.774900, -122.419400"), + ], + }); + + expect(body).toContain("QA_WHATSAPP_LOCATION_STREAM_OK"); + expect(body).not.toContain("QA_WHATSAPP_PREVIOUS_STREAM_OK"); + }); + it("uses image generation directives from request context when the latest user text is generic", async () => { const server = await startQaMockOpenAiServer({ host: "127.0.0.1", @@ -3482,6 +3668,66 @@ describe("qa mock openai server", () => { const ids = body.data.map((entry) => entry.id); expect(ids).toContain("claude-opus-4-8"); expect(ids).toContain("gpt-5.5"); + expect(ids).toContain("gpt-4o-transcribe"); + }); + + it("serves deterministic OpenAI-compatible audio transcription responses", async () => { + const server = await startQaMockOpenAiServer({ + host: "127.0.0.1", + port: 0, + }); + cleanups.push(async () => { + await server.stop(); + }); + + const response = await fetch(`${server.baseUrl}/v1/audio/transcriptions`, { + method: "POST", + headers: { + "content-type": "multipart/form-data; boundary=qa", + }, + body: "--qa\r\n--qa--\r\n", + }); + + expect(response.status).toBe(200); + await expect(response.json()).resolves.toEqual({ + text: "Reply with only this exact marker: WHATSAPP_QA_AUDIO_TRANSCRIPT_OK", + }); + }); + + it("serves deterministic WhatsApp group audio transcription for large audio uploads", async () => { + const server = await startQaMockOpenAiServer({ + host: "127.0.0.1", + port: 0, + }); + cleanups.push(async () => { + await server.stop(); + }); + + const triggered = await fetch(`${server.baseUrl}/v1/audio/transcriptions`, { + method: "POST", + headers: { + "content-type": "multipart/form-data; boundary=qa", + }, + body: `--qa\r\ncontent-disposition: form-data; name="file"; filename="upload.wav"\r\n\r\n${"x".repeat( + 64_000, + )}\r\n--qa--\r\n`, + }); + const quiet = await fetch(`${server.baseUrl}/v1/audio/transcriptions`, { + method: "POST", + headers: { + "content-type": "multipart/form-data; boundary=qa", + }, + body: '--qa\r\ncontent-disposition: form-data; name="file"; filename="upload.wav"\r\n\r\nx\r\n--qa--\r\n', + }); + + expect(triggered.status).toBe(200); + await expect(triggered.json()).resolves.toEqual({ + text: "openclawqa reply with only this exact marker after group audio preflight: WHATSAPP_QA_GROUP_AUDIO_TRANSCRIPT_OK", + }); + expect(quiet.status).toBe(200); + await expect(quiet.json()).resolves.toEqual({ + text: "Reply with only this exact marker: WHATSAPP_QA_AUDIO_TRANSCRIPT_OK", + }); }); it("dispatches an Anthropic /v1/messages read tool call for source discovery prompts", async () => { diff --git a/extensions/qa-lab/src/providers/mock-openai/server.ts b/extensions/qa-lab/src/providers/mock-openai/server.ts index a64a3fb4f4d1..6b343b499855 100644 --- a/extensions/qa-lab/src/providers/mock-openai/server.ts +++ b/extensions/qa-lab/src/providers/mock-openai/server.ts @@ -164,6 +164,7 @@ const QA_TELEGRAM_CURRENT_SESSION_STATUS_PROMPT_RE = /telegram current session_s const QA_TELEGRAM_STREAM_SINGLE_MARKER = "QA-TELEGRAM-STREAM-SINGLE-OK"; const QA_TELEGRAM_LONG_FINAL_THREE_CHUNK_PROMPT_RE = /telegram long final three chunk qa check/i; const QA_TELEGRAM_LONG_FINAL_PROMPT_RE = /telegram long final qa check/i; +const QA_WHATSAPP_LONG_FINAL_PROMPT_RE = /whatsapp long final qa check/i; const QA_SUBAGENT_DIRECT_FALLBACK_PROMPT_RE = /subagent direct fallback qa check/i; const QA_SUBAGENT_DIRECT_FALLBACK_WORKER_RE = /subagent direct fallback worker/i; const QA_SUBAGENT_DIRECT_FALLBACK_MARKER = "QA-SUBAGENT-DIRECT-FALLBACK-OK"; @@ -180,6 +181,11 @@ const QA_RELEASE_AUDIT_PROMPT_RE = /release readiness audit for the small projec const QA_TOOL_SEARCH_PROMPT_RE = /tool search qa check/i; const QA_TOOL_SEARCH_FAILURE_PROMPT_RE = /tool search qa failure/i; const QA_MCP_CODE_MODE_PROMPT_RE = /mcp code mode qa check/i; +const QA_AUDIO_TRANSCRIPTION_TEXT = + "Reply with only this exact marker: WHATSAPP_QA_AUDIO_TRANSCRIPT_OK"; +const QA_GROUP_AUDIO_TRANSCRIPTION_TEXT = + "openclawqa reply with only this exact marker after group audio preflight: WHATSAPP_QA_GROUP_AUDIO_TRANSCRIPT_OK"; +const QA_GROUP_AUDIO_MIN_MULTIPART_BODY_CHARS = 48_000; const QA_MCP_CODE_MODE_API_FILE_PROMPT_RE = /mcp code mode api file qa check/i; type MockScenarioState = { @@ -219,6 +225,13 @@ function readBody(req: IncomingMessage): Promise { }); } +function transcriptionTextForAudioRequest(rawBody: string) { + if (rawBody.length >= QA_GROUP_AUDIO_MIN_MULTIPART_BODY_CHARS) { + return QA_GROUP_AUDIO_TRANSCRIPTION_TEXT; + } + return QA_AUDIO_TRANSCRIPTION_TEXT; +} + function writeJson(res: ServerResponse, status: number, body: unknown) { const text = JSON.stringify(body); res.writeHead(status, { @@ -955,6 +968,33 @@ function extractExactMarkerDirective(text: string) { ); } +function extractWhatsAppLocationMarkerDirective(text: string) { + return extractLastCapture( + text, + /WhatsApp location marker:\s*([^\s`.,;:!?]+(?:-[^\s`.,;:!?]+)*)/i, + ); +} + +function extractWhatsAppContactMarkerDirective(text: string) { + return extractLastCapture(text, /WhatsApp contact marker:\s*([^\s`.,;:!?]+(?:-[^\s`.,;:!?]+)*)/i); +} + +function extractWhatsAppStickerMarkerDirective(text: string) { + return extractLastCapture(text, /WhatsApp sticker marker:\s*([^\s`.,;:!?]+(?:-[^\s`.,;:!?]+)*)/i); +} + +function shouldUseWhatsAppLocationMarker(prompt: string) { + return /(?:^|[\n:]\s*)📍\s*37\.774900,\s*-122\.419400\b/u.test(prompt.trim()); +} + +function shouldUseWhatsAppContactMarker(prompt: string) { + return /(?:^|[\n:]\s*))/iu.test(prompt.trim()); +} + +function shouldUseWhatsAppStickerMarker(prompt: string) { + return /(?:^|[\n:]\s*)(?:\s|$)/iu.test(prompt.trim()); +} + function extractLabeledMarkerDirective(text: string, label: string) { const escapedLabel = escapeRegExp(label); const backtickedMatch = extractLastCapture( @@ -1201,6 +1241,15 @@ function buildAssistantText( const exactReplyDirective = promptExactReplyDirective ?? extractExactReplyDirective(allInputText); const exactMarkerDirective = extractExactMarkerDirective(prompt) ?? extractExactMarkerDirective(allInputText); + const whatsAppLocationMarker = shouldUseWhatsAppLocationMarker(prompt) + ? extractWhatsAppLocationMarkerDirective(allInputText) + : ""; + const whatsAppContactMarker = shouldUseWhatsAppContactMarker(prompt) + ? extractWhatsAppContactMarkerDirective(allInputText) + : ""; + const whatsAppStickerMarker = shouldUseWhatsAppStickerMarker(prompt) + ? extractWhatsAppStickerMarkerDirective(allInputText) + : ""; const finishExactlyDirective = extractFinishExactlyDirective(prompt) ?? extractFinishExactlyDirective(allInputText); const latestImageUserTurn = extractLatestImageUserTurn(input); @@ -1242,6 +1291,15 @@ function buildAssistantText( ) { return "Protocol note: the attached image is split horizontally, with red on top and blue on the bottom."; } + if (whatsAppLocationMarker) { + return whatsAppLocationMarker; + } + if (whatsAppContactMarker) { + return whatsAppContactMarker; + } + if (whatsAppStickerMarker) { + return whatsAppStickerMarker; + } if (/\bmarker\b/i.test(allInputText) && exactReplyDirective) { return exactReplyDirective; } @@ -1566,19 +1624,20 @@ function splitMockStreamingText(text: string, parts = 3) { return chunks.length > 1 ? chunks : [text.slice(0, 1), text.slice(1)]; } -function buildTelegramLongFinalText({ +function buildQaLongFinalText({ endMarker = "TELEGRAM-LONG-FINAL-END", + segmentPrefix = "telegram-long-final-segment", segmentCount = 42, startMarker = "TELEGRAM-LONG-FINAL-BEGIN", }: { endMarker?: string; + segmentPrefix?: string; segmentCount?: number; startMarker?: string; } = {}) { const body = Array.from( { length: segmentCount }, - (_, index) => - `telegram-long-final-segment-${String(index + 1).padStart(3, "0")} ${"x".repeat(54)}`, + (_, index) => `${segmentPrefix}-${String(index + 1).padStart(3, "0")} ${"x".repeat(54)}`, ).join("\n"); return `${startMarker}\n${body}\n${endMarker}`; } @@ -1850,6 +1909,15 @@ async function buildResponsesPayload( extractExactReplyDirective(prompt) ?? extractExactReplyDirective(allInputText); const exactMarkerDirective = extractExactMarkerDirective(prompt) ?? extractExactMarkerDirective(allInputText); + const whatsAppLocationMarker = shouldUseWhatsAppLocationMarker(prompt) + ? extractWhatsAppLocationMarkerDirective(allInputText) + : ""; + const whatsAppContactMarker = shouldUseWhatsAppContactMarker(prompt) + ? extractWhatsAppContactMarkerDirective(allInputText) + : ""; + const whatsAppStickerMarker = shouldUseWhatsAppStickerMarker(prompt) + ? extractWhatsAppStickerMarkerDirective(allInputText) + : ""; const blockStreamingPrompt = extractLastMatchingUserText(extractAllUserTexts(input), QA_BLOCK_STREAMING_PROMPT_RE) || prompt || @@ -2073,7 +2141,7 @@ async function buildResponsesPayload( return buildAssistantEvents(""); } if (QA_TELEGRAM_LONG_FINAL_THREE_CHUNK_PROMPT_RE.test(allInputText)) { - const text = buildTelegramLongFinalText({ + const text = buildQaLongFinalText({ endMarker: "TELEGRAM-LONG-FINAL-3CHUNK-END", segmentCount: 96, startMarker: "TELEGRAM-LONG-FINAL-3CHUNK-BEGIN", @@ -2088,7 +2156,7 @@ async function buildResponsesPayload( ]); } if (QA_TELEGRAM_LONG_FINAL_PROMPT_RE.test(allInputText)) { - const text = buildTelegramLongFinalText(); + const text = buildQaLongFinalText(); return buildAssistantEvents([ { id: "msg_mock_telegram_long_final", @@ -2098,6 +2166,22 @@ async function buildResponsesPayload( }, ]); } + if (QA_WHATSAPP_LONG_FINAL_PROMPT_RE.test(allInputText)) { + const text = buildQaLongFinalText({ + endMarker: "WHATSAPP-LONG-FINAL-END", + segmentPrefix: "whatsapp-long-final-segment", + segmentCount: 64, + startMarker: "WHATSAPP-LONG-FINAL-BEGIN", + }); + return buildAssistantEvents([ + { + id: "msg_mock_whatsapp_long_final", + phase: "final_answer", + streamDeltas: splitMockStreamingText(text), + text, + }, + ]); + } if ( QA_STREAMING_PROMPT_RE.test(allInputText) && allInputText.includes(QA_TELEGRAM_STREAM_SINGLE_MARKER) @@ -2180,6 +2264,15 @@ async function buildResponsesPayload( exactMarkerDirective ?? exactReplyDirective ?? "QA-GROUP-FALLBACK-OK", ); } + if (whatsAppLocationMarker) { + return buildAssistantEvents(whatsAppLocationMarker); + } + if (whatsAppContactMarker) { + return buildAssistantEvents(whatsAppContactMarker); + } + if (whatsAppStickerMarker) { + return buildAssistantEvents(whatsAppStickerMarker); + } if (/\bmarker\b/i.test(prompt) && exactReplyDirective) { return buildAssistantEvents(exactReplyDirective); } @@ -3196,6 +3289,7 @@ export async function startQaMockOpenAiServer(params?: { host?: string; port?: n { id: "gpt-5.5", object: "model" }, { id: "gpt-5.5-alt", object: "model" }, { id: "gpt-image-1", object: "model" }, + { id: "gpt-4o-transcribe", object: "model" }, { id: "text-embedding-3-small", object: "model" }, { id: "claude-opus-4-8", object: "model" }, { id: "claude-sonnet-4-6", object: "model" }, @@ -3232,6 +3326,13 @@ export async function startQaMockOpenAiServer(params?: { host?: string; port?: n }); return; } + if (req.method === "POST" && url.pathname === "/v1/audio/transcriptions") { + const raw = await readBody(req); + writeJson(res, 200, { + text: transcriptionTextForAudioRequest(raw), + }); + return; + } if (req.method === "POST" && url.pathname === "/v1/embeddings") { const raw = await readBody(req); const body = raw ? (JSON.parse(raw) as Record) : {}; diff --git a/extensions/whatsapp/src/auto-reply.test-harness.ts b/extensions/whatsapp/src/auto-reply.test-harness.ts index f077fdd507dc..91e3155b686d 100644 --- a/extensions/whatsapp/src/auto-reply.test-harness.ts +++ b/extensions/whatsapp/src/auto-reply.test-harness.ts @@ -30,6 +30,9 @@ type MockWebListener = { signalClose: () => void; sendMessage: () => Promise; sendPoll: () => Promise; + sendContact: () => Promise; + sendLocation: () => Promise; + sendSticker: () => Promise; sendReaction: () => Promise; sendComposingTo: () => Promise; }; @@ -264,6 +267,9 @@ export function createMockWebListener(): MockWebListener { signalClose: vi.fn(), sendMessage: vi.fn(async () => createAcceptedWhatsAppSendResult("text", "msg-1")), sendPoll: vi.fn(async () => createAcceptedWhatsAppSendResult("poll", "poll-1")), + sendContact: vi.fn(async () => createAcceptedWhatsAppSendResult("contact", "contact-1")), + sendLocation: vi.fn(async () => createAcceptedWhatsAppSendResult("location", "location-1")), + sendSticker: vi.fn(async () => createAcceptedWhatsAppSendResult("sticker", "sticker-1")), sendReaction: vi.fn(async () => createAcceptedWhatsAppSendResult("reaction", "reaction-1")), sendComposingTo: vi.fn(async () => undefined), }; diff --git a/extensions/whatsapp/src/inbound/media-mimetype.ts b/extensions/whatsapp/src/inbound/media-mimetype.ts new file mode 100644 index 000000000000..607972e1fe3a --- /dev/null +++ b/extensions/whatsapp/src/inbound/media-mimetype.ts @@ -0,0 +1,33 @@ +// Whatsapp plugin module implements inbound media MIME normalization. +import type { proto } from "baileys"; + +/** + * Resolve the MIME type for an inbound media message. + * Falls back to WhatsApp's standard formats when Baileys omits the MIME. + */ +export function resolveInboundMediaMimetype(message: proto.IMessage): string | undefined { + const explicit = + message.imageMessage?.mimetype ?? + message.videoMessage?.mimetype ?? + message.documentMessage?.mimetype ?? + message.audioMessage?.mimetype ?? + message.stickerMessage?.mimetype ?? + undefined; + if (explicit) { + return explicit; + } + // WhatsApp voice messages (PTT) and audio use OGG Opus by default. + if (message.audioMessage) { + return "audio/ogg; codecs=opus"; + } + if (message.imageMessage) { + return "image/jpeg"; + } + if (message.videoMessage) { + return "video/mp4"; + } + if (message.stickerMessage) { + return "image/webp"; + } + return undefined; +} diff --git a/extensions/whatsapp/src/inbound/media.ts b/extensions/whatsapp/src/inbound/media.ts index 3cb3c77b5c0b..2b8dc6ec381d 100644 --- a/extensions/whatsapp/src/inbound/media.ts +++ b/extensions/whatsapp/src/inbound/media.ts @@ -4,6 +4,7 @@ import { saveMediaStream, type SavedMedia } from "openclaw/plugin-sdk/media-stor import { logVerbose } from "openclaw/plugin-sdk/runtime-env"; import type { createWaSocket } from "../session.js"; import { extractContextInfo } from "./extract.js"; +import { resolveInboundMediaMimetype } from "./media-mimetype.js"; import { downloadMediaMessage, normalizeMessageContent } from "./runtime-api.js"; export class WhatsAppInboundMediaLimitExceededError extends Error { @@ -18,37 +19,6 @@ function unwrapMessage(message: proto.IMessage | undefined): proto.IMessage | un return normalized; } -/** - * Resolve the MIME type for an inbound media message. - * Falls back to WhatsApp's standard formats when Baileys omits the MIME. - */ -function resolveMediaMimetype(message: proto.IMessage): string | undefined { - const explicit = - message.imageMessage?.mimetype ?? - message.videoMessage?.mimetype ?? - message.documentMessage?.mimetype ?? - message.audioMessage?.mimetype ?? - message.stickerMessage?.mimetype ?? - undefined; - if (explicit) { - return explicit; - } - // WhatsApp voice messages (PTT) and audio use OGG Opus by default - if (message.audioMessage) { - return "audio/ogg; codecs=opus"; - } - if (message.imageMessage) { - return "image/jpeg"; - } - if (message.videoMessage) { - return "video/mp4"; - } - if (message.stickerMessage) { - return "image/webp"; - } - return undefined; -} - export async function downloadInboundMedia( msg: proto.IWebMessageInfo, sock: Awaited>, @@ -58,7 +28,7 @@ export async function downloadInboundMedia( if (!message) { return undefined; } - const mimetype = resolveMediaMimetype(message); + const mimetype = resolveInboundMediaMimetype(message); const fileName = message.documentMessage?.fileName ?? undefined; if ( !message.imageMessage && diff --git a/extensions/whatsapp/src/inbound/monitor.ts b/extensions/whatsapp/src/inbound/monitor.ts index 3f60e977a5ac..d87938e89345 100644 --- a/extensions/whatsapp/src/inbound/monitor.ts +++ b/extensions/whatsapp/src/inbound/monitor.ts @@ -1289,8 +1289,10 @@ export async function attachWebInboxToSocket( signalClose: (reason?: WebListenerCloseReason) => { resolveClose(reason ?? { status: undefined, isLoggedOut: false, error: "closed" }); }, - // IPC surface (sendMessage/sendPoll/sendReaction/sendComposingTo) - ...sendApi, + sendComposingTo: sendApi.sendComposingTo, + sendMessage: sendApi.sendMessage, + sendPoll: sendApi.sendPoll, + sendReaction: sendApi.sendReaction, } as const; } diff --git a/extensions/whatsapp/src/inbound/send-api.test.ts b/extensions/whatsapp/src/inbound/send-api.test.ts index 4d172c6e67e4..852d4dd01c10 100644 --- a/extensions/whatsapp/src/inbound/send-api.test.ts +++ b/extensions/whatsapp/src/inbound/send-api.test.ts @@ -7,6 +7,7 @@ import { listMessageReceiptPlatformIds } from "openclaw/plugin-sdk/channel-outbo import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { resolveWhatsAppOutboundMentions } from "./outbound-mentions.js"; import { createWebSendApi } from "./send-api.js"; +import type { WhatsAppSendResult } from "./send-result.js"; const recordChannelActivity = vi.hoisted(() => vi.fn()); const imageOps = vi.hoisted(() => ({ @@ -105,10 +106,7 @@ describe("createWebSendApi", () => { expectRecordFields(requireSendContent(callIndex), fields); } - function expectSendResultFields( - result: Awaited>, - fields: Record, - ) { + function expectSendResultFields(result: WhatsAppSendResult, fields: Record) { expectRecordFields(requireRecord(result, "send result"), fields); } @@ -229,6 +227,72 @@ describe("createWebSendApi", () => { }); }); + it("sends structured contact messages through the canonical send path", async () => { + const res = await api.sendContact("+1555", { + displayName: "QA Contact", + vcard: "BEGIN:VCARD\nFN:QA Contact\nEND:VCARD", + }); + + expect(sendMessage).toHaveBeenCalledWith("1555@s.whatsapp.net", { + contacts: { + displayName: "QA Contact", + contacts: [ + { + displayName: "QA Contact", + vcard: "BEGIN:VCARD\nFN:QA Contact\nEND:VCARD", + }, + ], + }, + }); + expectSendResultFields(res, { + kind: "contact", + messageId: "msg-1", + providerAccepted: true, + }); + expect(recordChannelActivity).toHaveBeenCalledWith({ + channel: "whatsapp", + accountId: "main", + direction: "outbound", + }); + }); + + it("sends structured location messages through the canonical send path", async () => { + const res = await api.sendLocation("+1555", { + degreesLatitude: 37.7749, + degreesLongitude: -122.4194, + name: "QA Location", + }); + + expect(sendMessage).toHaveBeenCalledWith("1555@s.whatsapp.net", { + location: { + address: undefined, + degreesLatitude: 37.7749, + degreesLongitude: -122.4194, + name: "QA Location", + }, + }); + expectSendResultFields(res, { + kind: "location", + messageId: "msg-1", + providerAccepted: true, + }); + }); + + it("sends structured sticker messages through the canonical send path", async () => { + const payload = Buffer.from("webp"); + const res = await api.sendSticker("+1555", payload); + + expect(sendMessage).toHaveBeenCalledWith("1555@s.whatsapp.net", { + sticker: payload, + mimetype: "image/webp", + }); + expectSendResultFields(res, { + kind: "sticker", + messageId: "msg-1", + providerAccepted: true, + }); + }); + it("adds native mention metadata to group text sends", async () => { api = createWebSendApi({ sock: { sendMessage, sendPresenceUpdate }, diff --git a/extensions/whatsapp/src/inbound/send-api.ts b/extensions/whatsapp/src/inbound/send-api.ts index 64cd3b5b01f9..a7677466e276 100644 --- a/extensions/whatsapp/src/inbound/send-api.ts +++ b/extensions/whatsapp/src/inbound/send-api.ts @@ -18,10 +18,27 @@ import { import { combineWhatsAppSendResults, normalizeWhatsAppSendResult, + type WhatsAppSendKind, type WhatsAppSendResult, } from "./send-result.js"; import type { ActiveWebSendOptions } from "./types.js"; +type StructuredContactSend = { + displayName: string; + vcard: string; +}; + +type StructuredLocationSend = { + address?: string; + degreesLatitude: number; + degreesLongitude: number; + name?: string; +}; + +type StructuredStickerSendOptions = { + mimetype?: string; +}; + function recordWhatsAppOutbound(accountId: string) { recordChannelActivity({ channel: "whatsapp", @@ -65,6 +82,16 @@ export function createWebSendApi(params: { params.resolveOutboundMentions ? await params.resolveOutboundMentions({ jid, text }) : { text, mentionedJids: [] }; + const sendStructuredMessage = async ( + to: string, + content: AnyMessageContent, + kind: WhatsAppSendKind, + ): Promise => { + const jid = resolveOutboundJid(to); + const result = await params.sock.sendMessage(jid, content); + recordWhatsAppOutbound(params.defaultAccountId); + return normalizeWhatsAppSendResult(result, kind); + }; return { sendMessage: async ( @@ -160,16 +187,68 @@ export function createWebSendApi(params: { to: string, poll: { question: string; options: string[]; maxSelections?: number }, ): Promise => { - const jid = resolveOutboundJid(to); - const result = await params.sock.sendMessage(jid, { - poll: { - name: poll.question, - values: poll.options, - selectableCount: poll.maxSelections ?? 1, - }, - } as AnyMessageContent); - recordWhatsAppOutbound(params.defaultAccountId); - return normalizeWhatsAppSendResult(result, "poll"); + return await sendStructuredMessage( + to, + { + poll: { + name: poll.question, + values: poll.options, + selectableCount: poll.maxSelections ?? 1, + }, + } as AnyMessageContent, + "poll", + ); + }, + sendContact: async ( + to: string, + contact: StructuredContactSend, + ): Promise => { + return await sendStructuredMessage( + to, + { + contacts: { + displayName: contact.displayName, + contacts: [ + { + displayName: contact.displayName, + vcard: contact.vcard, + }, + ], + }, + } as AnyMessageContent, + "contact", + ); + }, + sendLocation: async ( + to: string, + location: StructuredLocationSend, + ): Promise => { + return await sendStructuredMessage( + to, + { + location: { + degreesLatitude: location.degreesLatitude, + degreesLongitude: location.degreesLongitude, + name: location.name, + address: location.address, + }, + } as AnyMessageContent, + "location", + ); + }, + sendSticker: async ( + to: string, + stickerBuffer: Buffer, + options?: StructuredStickerSendOptions, + ): Promise => { + return await sendStructuredMessage( + to, + { + sticker: stickerBuffer, + mimetype: options?.mimetype ?? "image/webp", + } as AnyMessageContent, + "sticker", + ); }, sendReaction: async ( chatJid: string, diff --git a/extensions/whatsapp/src/inbound/send-result.ts b/extensions/whatsapp/src/inbound/send-result.ts index 231d2800524c..a66c10a1d392 100644 --- a/extensions/whatsapp/src/inbound/send-result.ts +++ b/extensions/whatsapp/src/inbound/send-result.ts @@ -9,7 +9,14 @@ import { } from "openclaw/plugin-sdk/channel-outbound"; import { normalizeStringEntries, uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime"; -export type WhatsAppSendKind = "media" | "poll" | "reaction" | "text"; +export type WhatsAppSendKind = + | "contact" + | "location" + | "media" + | "poll" + | "reaction" + | "sticker" + | "text"; type WhatsAppSendKey = Omit< Pick, diff --git a/extensions/whatsapp/src/qa-driver.runtime.test.ts b/extensions/whatsapp/src/qa-driver.runtime.test.ts index 37434964aaa0..5b824fb5eee4 100644 --- a/extensions/whatsapp/src/qa-driver.runtime.test.ts +++ b/extensions/whatsapp/src/qa-driver.runtime.test.ts @@ -7,12 +7,21 @@ import { startWhatsAppQaDriverSession } from "./qa-driver.runtime.js"; const mocks = vi.hoisted(() => ({ createWaSocket: vi.fn(), jidToE164: vi.fn(), + sendContact: vi.fn(), + sendLocation: vi.fn(), + sendPoll: vi.fn(), + sendReaction: vi.fn(), + sendSticker: vi.fn(), sendMessage: vi.fn(), + socketSendMessage: vi.fn(), waitForWaConnection: vi.fn(), })); vi.mock("./session.js", () => ({ createWaSocket: mocks.createWaSocket, + formatError: (error: unknown) => (error instanceof Error ? error.message : String(error)), + getStatusCode: (error: unknown) => + (error as { output?: { statusCode?: number } } | undefined)?.output?.statusCode, waitForWaConnection: mocks.waitForWaConnection, })); @@ -22,7 +31,12 @@ vi.mock("./text-runtime.js", () => ({ vi.mock("./inbound/send-api.js", () => ({ createWebSendApi: () => ({ + sendContact: mocks.sendContact, + sendLocation: mocks.sendLocation, sendMessage: mocks.sendMessage, + sendPoll: mocks.sendPoll, + sendReaction: mocks.sendReaction, + sendSticker: mocks.sendSticker, }), })); @@ -30,17 +44,18 @@ function createMockSocket() { return { end: vi.fn(), ev: new EventEmitter(), + sendMessage: mocks.socketSendMessage, ws: { close: vi.fn(), }, }; } -function incomingMessage(remoteJid: string, text: string): WAMessage { +function incomingMessage(remoteJid: string, text: string, id = "message-1"): WAMessage { return { key: { fromMe: false, - id: "message-1", + id, remoteJid, }, message: { @@ -49,6 +64,165 @@ function incomingMessage(remoteJid: string, text: string): WAMessage { } as WAMessage; } +function incomingImageMessage(remoteJid: string, text: string): WAMessage { + return { + key: { + fromMe: false, + id: "image-1", + remoteJid, + }, + message: { + imageMessage: { + caption: text, + mimetype: "image/png", + }, + }, + } as WAMessage; +} + +function incomingImageMessageWithoutMime(remoteJid: string): WAMessage { + return { + key: { + fromMe: false, + id: "image-no-mime-1", + remoteJid, + }, + message: { + imageMessage: {}, + }, + } as WAMessage; +} + +function incomingStickerMessageWithoutMime(remoteJid: string): WAMessage { + return { + key: { + fromMe: false, + id: "sticker-no-mime-1", + remoteJid, + }, + message: { + stickerMessage: {}, + }, + } as WAMessage; +} + +function incomingAudioMessage(remoteJid: string): WAMessage { + return { + key: { + fromMe: false, + id: "audio-1", + remoteJid, + }, + message: { + audioMessage: { + mimetype: "audio/ogg; codecs=opus", + }, + }, + } as WAMessage; +} + +function incomingEditedImageMessage(remoteJid: string): WAMessage { + return { + key: { + fromMe: false, + id: "edited-image-1", + remoteJid, + }, + message: { + editedMessage: { + message: { + imageMessage: { + caption: "edited image caption", + }, + }, + }, + }, + } as WAMessage; +} + +function incomingLocationMessage(remoteJid: string): WAMessage { + return { + key: { + fromMe: false, + id: "location-1", + remoteJid, + }, + message: { + locationMessage: { + degreesLatitude: 37.7749, + degreesLongitude: -122.4194, + }, + }, + } as WAMessage; +} + +function incomingReactionMessage(remoteJid: string): WAMessage { + return { + key: { + fromMe: false, + id: "reaction-1", + remoteJid, + }, + message: { + reactionMessage: { + text: "👍", + key: { + fromMe: true, + id: "driver-message-1", + participant: "15551234567@s.whatsapp.net", + }, + }, + }, + } as WAMessage; +} + +function incomingQuotedMessage(remoteJid: string): WAMessage { + return { + key: { + fromMe: false, + id: "quoted-reply-1", + remoteJid, + }, + message: { + extendedTextMessage: { + text: "reply body", + contextInfo: { + participant: "15551234567@s.whatsapp.net", + quotedMessage: { + conversation: "original body", + }, + stanzaId: "driver-message-1", + }, + }, + }, + } as WAMessage; +} + +function incomingQuotedLocationMessage(remoteJid: string): WAMessage { + return { + key: { + fromMe: false, + id: "quoted-location-reply-1", + remoteJid, + }, + message: { + extendedTextMessage: { + text: "reply body", + contextInfo: { + participant: "15551234567@s.whatsapp.net", + quotedMessage: { + locationMessage: { + degreesLatitude: 37.7749, + degreesLongitude: -122.4194, + }, + }, + stanzaId: "driver-location-1", + }, + }, + }, + } as WAMessage; +} + describe("startWhatsAppQaDriverSession", () => { afterEach(() => { vi.useRealTimers(); @@ -79,6 +253,7 @@ describe("startWhatsAppQaDriverSession", () => { { fromJid: "12345@lid", fromPhoneE164: "+15551234567", + kind: "text", messageId: "message-1", observedAt, text: "hello", @@ -88,6 +263,349 @@ describe("startWhatsAppQaDriverSession", () => { await session.close(); }); + it("does not satisfy a wait with messages observed before the lower bound", async () => { + vi.useFakeTimers(); + const sock = createMockSocket(); + mocks.createWaSocket.mockResolvedValue(sock); + mocks.waitForWaConnection.mockResolvedValue(undefined); + mocks.jidToE164.mockReturnValue("+15551234567"); + + vi.setSystemTime(new Date("2026-06-04T23:42:32.036Z")); + const session = await startWhatsAppQaDriverSession({ + authDir: "/tmp/openclaw-whatsapp-auth", + }); + + sock.ev.emit("messages.upsert", { + messages: [incomingMessage("12345@lid", "OpenClaw status stale", "stale-message")], + }); + + const observedAfter = new Date("2026-06-04T23:46:59.166Z"); + vi.setSystemTime(observedAfter); + const waited = session.waitForMessage({ + observedAfter, + timeoutMs: 1_000, + match: (message) => message.text.includes("OpenClaw status"), + }); + + vi.setSystemTime(new Date("2026-06-04T23:47:00.000Z")); + sock.ev.emit("messages.upsert", { + messages: [incomingMessage("12345@lid", "OpenClaw status fresh", "fresh-message")], + }); + + await expect(waited).resolves.toMatchObject({ + messageId: "fresh-message", + text: "OpenClaw status fresh", + }); + + await session.close(); + }); + + it("observes media messages without dropping their caption text", async () => { + const sock = createMockSocket(); + mocks.createWaSocket.mockResolvedValue(sock); + mocks.waitForWaConnection.mockResolvedValue(undefined); + mocks.jidToE164.mockReturnValue("+15551234567"); + + const session = await startWhatsAppQaDriverSession({ + authDir: "/tmp/openclaw-whatsapp-auth", + }); + + sock.ev.emit("messages.upsert", { + messages: [incomingImageMessage("12345@lid", "image caption")], + }); + + expect(session.getObservedMessages()[0]).toMatchObject({ + hasMedia: true, + kind: "media", + mediaType: "image/png", + text: "image caption", + }); + + await session.close(); + }); + + it("observes audio media messages without requiring a text body", async () => { + const sock = createMockSocket(); + mocks.createWaSocket.mockResolvedValue(sock); + mocks.waitForWaConnection.mockResolvedValue(undefined); + mocks.jidToE164.mockReturnValue("+15551234567"); + + const session = await startWhatsAppQaDriverSession({ + authDir: "/tmp/openclaw-whatsapp-auth", + }); + + sock.ev.emit("messages.upsert", { + messages: [incomingAudioMessage("12345@lid")], + }); + + expect(session.getObservedMessages()[0]).toMatchObject({ + hasMedia: true, + kind: "media", + mediaType: "audio/ogg; codecs=opus", + text: "", + }); + + await session.close(); + }); + + it("uses canonical WhatsApp media MIME defaults when Baileys omits MIME", async () => { + const sock = createMockSocket(); + mocks.createWaSocket.mockResolvedValue(sock); + mocks.waitForWaConnection.mockResolvedValue(undefined); + mocks.jidToE164.mockReturnValue("+15551234567"); + + const session = await startWhatsAppQaDriverSession({ + authDir: "/tmp/openclaw-whatsapp-auth", + }); + + sock.ev.emit("messages.upsert", { + messages: [ + incomingImageMessageWithoutMime("12345@lid"), + incomingStickerMessageWithoutMime("12345@lid"), + ], + }); + + expect(session.getObservedMessages()).toMatchObject([ + { + hasMedia: true, + kind: "media", + mediaType: "image/jpeg", + }, + { + hasMedia: true, + kind: "media", + mediaType: "image/webp", + }, + ]); + + await session.close(); + }); + + it("observes media through Baileys future-proof wrappers", async () => { + const sock = createMockSocket(); + mocks.createWaSocket.mockResolvedValue(sock); + mocks.waitForWaConnection.mockResolvedValue(undefined); + mocks.jidToE164.mockReturnValue("+15551234567"); + + const session = await startWhatsAppQaDriverSession({ + authDir: "/tmp/openclaw-whatsapp-auth", + }); + + sock.ev.emit("messages.upsert", { + messages: [incomingEditedImageMessage("12345@lid")], + }); + + expect(session.getObservedMessages()[0]).toMatchObject({ + hasMedia: true, + kind: "media", + mediaType: "image/jpeg", + text: "edited image caption", + }); + + await session.close(); + }); + + it("observes top-level location messages with canonical location text", async () => { + const sock = createMockSocket(); + mocks.createWaSocket.mockResolvedValue(sock); + mocks.waitForWaConnection.mockResolvedValue(undefined); + mocks.jidToE164.mockReturnValue("+15551234567"); + + const session = await startWhatsAppQaDriverSession({ + authDir: "/tmp/openclaw-whatsapp-auth", + }); + + sock.ev.emit("messages.upsert", { + messages: [incomingLocationMessage("12345@lid")], + }); + + expect(session.getObservedMessages()[0]).toMatchObject({ + kind: "location", + text: "📍 37.774900, -122.419400", + }); + + await session.close(); + }); + + it("observes reaction messages that have no text body", async () => { + const sock = createMockSocket(); + mocks.createWaSocket.mockResolvedValue(sock); + mocks.waitForWaConnection.mockResolvedValue(undefined); + mocks.jidToE164.mockReturnValue("+15551234567"); + + const session = await startWhatsAppQaDriverSession({ + authDir: "/tmp/openclaw-whatsapp-auth", + }); + + sock.ev.emit("messages.upsert", { + messages: [incomingReactionMessage("12345@lid")], + }); + + expect(session.getObservedMessages()[0]).toMatchObject({ + kind: "reaction", + reaction: { + emoji: "👍", + fromMe: true, + messageId: "driver-message-1", + participant: "15551234567@s.whatsapp.net", + }, + text: "", + }); + + await session.close(); + }); + + it("observes quoted reply context", async () => { + const sock = createMockSocket(); + mocks.createWaSocket.mockResolvedValue(sock); + mocks.waitForWaConnection.mockResolvedValue(undefined); + mocks.jidToE164.mockReturnValue("+15551234567"); + + const session = await startWhatsAppQaDriverSession({ + authDir: "/tmp/openclaw-whatsapp-auth", + }); + + sock.ev.emit("messages.upsert", { + messages: [incomingQuotedMessage("12345@lid")], + }); + + expect(session.getObservedMessages()[0]).toMatchObject({ + kind: "text", + quoted: { + messageId: "driver-message-1", + participant: "15551234567@s.whatsapp.net", + text: "original body", + }, + text: "reply body", + }); + + await session.close(); + }); + + it("observes quoted location context with canonical reply body text", async () => { + const sock = createMockSocket(); + mocks.createWaSocket.mockResolvedValue(sock); + mocks.waitForWaConnection.mockResolvedValue(undefined); + mocks.jidToE164.mockReturnValue("+15551234567"); + + const session = await startWhatsAppQaDriverSession({ + authDir: "/tmp/openclaw-whatsapp-auth", + }); + + sock.ev.emit("messages.upsert", { + messages: [incomingQuotedLocationMessage("12345@lid")], + }); + + expect(session.getObservedMessages()[0]).toMatchObject({ + kind: "text", + quoted: { + messageId: "driver-location-1", + participant: "15551234567@s.whatsapp.net", + text: "📍 37.774900, -122.419400", + }, + text: "reply body", + }); + + await session.close(); + }); + + it("uses the web send API for existing outbound helpers", async () => { + const sock = createMockSocket(); + mocks.createWaSocket.mockResolvedValue(sock); + mocks.waitForWaConnection.mockResolvedValue(undefined); + mocks.sendMessage.mockResolvedValue({ messageId: "send-1" }); + mocks.sendPoll.mockResolvedValue({ messageId: "poll-1" }); + mocks.sendReaction.mockResolvedValue({ messageId: "reaction-send-1" }); + + const session = await startWhatsAppQaDriverSession({ + authDir: "/tmp/openclaw-whatsapp-auth", + }); + + await expect( + session.sendMedia("15551234567", "caption", Buffer.from("png"), "image/png", { + fileName: "qa.png", + }), + ).resolves.toEqual({ messageId: "send-1" }); + await expect( + session.sendPoll("15551234567", { + question: "Pick one", + options: ["A", "B"], + }), + ).resolves.toEqual({ messageId: "poll-1" }); + await expect( + session.sendReaction("15551234567@s.whatsapp.net", "driver-message-1", "👍", { + fromMe: true, + }), + ).resolves.toEqual({ messageId: "reaction-send-1" }); + + expect(mocks.sendMessage).toHaveBeenCalledWith( + "15551234567", + "caption", + Buffer.from("png"), + "image/png", + { fileName: "qa.png" }, + ); + expect(mocks.sendPoll).toHaveBeenCalledWith("15551234567", { + question: "Pick one", + options: ["A", "B"], + }); + expect(mocks.sendReaction).toHaveBeenCalledWith( + "15551234567@s.whatsapp.net", + "driver-message-1", + "👍", + true, + undefined, + ); + + await session.close(); + }); + + it("sends structured QA stimuli through the web send API", async () => { + const sock = createMockSocket(); + mocks.createWaSocket.mockResolvedValue(sock); + mocks.waitForWaConnection.mockResolvedValue(undefined); + mocks.sendContact.mockResolvedValue({ messageId: "contact-1" }); + mocks.sendLocation.mockResolvedValue({ messageId: "location-1" }); + mocks.sendSticker.mockResolvedValue({ messageId: "sticker-1" }); + + const session = await startWhatsAppQaDriverSession({ + authDir: "/tmp/openclaw-whatsapp-auth", + }); + + await expect( + session.sendContact("15551234567", { + displayName: "QA Contact", + vcard: "BEGIN:VCARD\nFN:QA Contact\nEND:VCARD", + }), + ).resolves.toEqual({ messageId: "contact-1" }); + await expect( + session.sendLocation("15551234567", { + degreesLatitude: 37.7749, + degreesLongitude: -122.4194, + name: "QA Location", + }), + ).resolves.toEqual({ messageId: "location-1" }); + await expect( + session.sendSticker("15551234567", Buffer.from("webp"), { mimetype: "image/webp" }), + ).resolves.toEqual({ messageId: "sticker-1" }); + + expect(mocks.sendContact).toHaveBeenCalledWith("15551234567", { + displayName: "QA Contact", + vcard: "BEGIN:VCARD\nFN:QA Contact\nEND:VCARD", + }); + expect(mocks.sendLocation).toHaveBeenCalledWith("15551234567", { + degreesLatitude: 37.7749, + degreesLongitude: -122.4194, + name: "QA Location", + }); + expect(mocks.sendSticker).toHaveBeenCalledWith("15551234567", Buffer.from("webp"), { + mimetype: "image/webp", + }); + expect(mocks.socketSendMessage).not.toHaveBeenCalled(); + + await session.close(); + }); + it("passes the connection timeout to the shared connection waiter", async () => { const sock = createMockSocket(); mocks.createWaSocket.mockResolvedValue(sock); @@ -103,6 +621,73 @@ describe("startWhatsAppQaDriverSession", () => { await session.close(); }); + it("can wait for pending notifications before returning the driver session", async () => { + const sock = createMockSocket(); + mocks.createWaSocket.mockResolvedValue(sock); + mocks.waitForWaConnection.mockResolvedValue(undefined); + + const pending = startWhatsAppQaDriverSession({ + authDir: "/tmp/openclaw-whatsapp-auth", + connectionTimeoutMs: 10_000, + waitForPendingNotifications: true, + }); + let settled = false; + pending.then( + () => { + settled = true; + }, + () => { + settled = true; + }, + ); + + await Promise.resolve(); + expect(settled).toBe(false); + + sock.ev.emit("connection.update", { receivedPendingNotifications: true }); + + const session = await pending; + expect(settled).toBe(true); + await session.close(); + }); + + it("rejects pending and future waits when the connected driver session closes", async () => { + const sock = createMockSocket(); + mocks.createWaSocket.mockResolvedValue(sock); + mocks.waitForWaConnection.mockResolvedValue(undefined); + + const session = await startWhatsAppQaDriverSession({ + authDir: "/tmp/openclaw-whatsapp-auth", + }); + const pending = session.waitForMessage({ + match: (message) => message.text.includes("approval required"), + timeoutMs: 60_000, + }); + + sock.ev.emit("connection.update", { + connection: "close", + lastDisconnect: { + date: new Date("2026-06-05T17:54:52.000Z"), + error: { + output: { + statusCode: 428, + }, + }, + }, + }); + + await expect(pending).rejects.toThrow("WhatsApp QA driver connection closed (status 428)"); + await expect( + session.waitForMessage({ + match: (message) => message.text.includes("approval required"), + timeoutMs: 60_000, + }), + ).rejects.toThrow("WhatsApp QA driver connection closed (status 428)"); + expect(sock.ev.listenerCount("messages.upsert")).toBe(0); + expect(sock.ev.listenerCount("connection.update")).toBe(0); + expect(sock.end).toHaveBeenCalledOnce(); + }); + it("closes the socket and removes listeners when connection setup times out", async () => { const sock = createMockSocket(); const timeoutError = new Error("timed out waiting for WhatsApp QA driver session"); @@ -118,6 +703,7 @@ describe("startWhatsAppQaDriverSession", () => { expect(mocks.waitForWaConnection).toHaveBeenCalledWith(sock, { timeoutMs: 10 }); expect(sock.ev.listenerCount("messages.upsert")).toBe(0); + expect(sock.ev.listenerCount("connection.update")).toBe(0); expect(sock.end).toHaveBeenCalledOnce(); }); }); diff --git a/extensions/whatsapp/src/qa-driver.runtime.ts b/extensions/whatsapp/src/qa-driver.runtime.ts index 0d4d1464e644..45d7c831b9b9 100644 --- a/extensions/whatsapp/src/qa-driver.runtime.ts +++ b/extensions/whatsapp/src/qa-driver.runtime.ts @@ -1,24 +1,118 @@ // Whatsapp plugin module implements qa driver behavior. -import type { WAMessage } from "baileys"; -import { extractText } from "./inbound/extract.js"; +import type { ConnectionState, proto, WAMessage } from "baileys"; +import { formatLocationText } from "openclaw/plugin-sdk/channel-inbound"; +import { + describeReplyContext, + extractContextInfo, + extractLocationData, + extractText, +} from "./inbound/extract.js"; +import { resolveInboundMediaMimetype } from "./inbound/media-mimetype.js"; +import { normalizeMessageContent } from "./inbound/runtime-api.js"; import { createWebSendApi } from "./inbound/send-api.js"; -import { createWaSocket, waitForWaConnection } from "./session.js"; +import type { ActiveWebSendOptions } from "./inbound/types.js"; +import { createWaSocket, formatError, getStatusCode, waitForWaConnection } from "./session.js"; import { jidToE164 } from "./text-runtime.js"; +export type WhatsAppQaDriverObservedMessageKind = + | "media" + | "location" + | "poll" + | "reaction" + | "text" + | "unknown"; + +export type WhatsAppQaDriverQuotedMessage = { + messageId?: string; + participant?: string; + text?: string; +}; + +export type WhatsAppQaDriverObservedReaction = { + emoji: string; + fromMe?: boolean; + messageId?: string; + participant?: string; +}; + +export type WhatsAppQaDriverObservedPoll = { + options: string[]; + question?: string; +}; + export type WhatsAppQaDriverObservedMessage = { fromJid?: string; fromPhoneE164?: string | null; + hasMedia?: boolean; + kind: WhatsAppQaDriverObservedMessageKind; + mediaFileName?: string; + mediaType?: string; messageId?: string; observedAt: string; + poll?: WhatsAppQaDriverObservedPoll; + quoted?: WhatsAppQaDriverQuotedMessage; + reaction?: WhatsAppQaDriverObservedReaction; text: string; }; +export type WhatsAppQaDriverSendTextOptions = Pick; + +export type WhatsAppQaDriverSendMediaOptions = Pick< + ActiveWebSendOptions, + "asDocument" | "fileName" | "gifPlayback" | "quotedMessageKey" +>; + +export type WhatsAppQaDriverSendReactionOptions = { + fromMe: boolean; + participant?: string; +}; + export type WhatsAppQaDriverSession = { close: () => Promise; getObservedMessages: () => WhatsAppQaDriverObservedMessage[]; - sendText: (to: string, text: string) => Promise<{ messageId?: string }>; + sendContact: ( + to: string, + contact: { displayName: string; vcard: string }, + ) => Promise<{ messageId?: string }>; + sendLocation: ( + to: string, + location: { + address?: string; + degreesLatitude: number; + degreesLongitude: number; + name?: string; + }, + ) => Promise<{ messageId?: string }>; + sendMedia: ( + to: string, + text: string, + mediaBuffer: Buffer, + mediaType: string, + options?: WhatsAppQaDriverSendMediaOptions, + ) => Promise<{ messageId?: string }>; + sendPoll: ( + to: string, + poll: { maxSelections?: number; options: string[]; question: string }, + ) => Promise<{ messageId?: string }>; + sendReaction: ( + chatJid: string, + messageId: string, + emoji: string, + options: WhatsAppQaDriverSendReactionOptions, + ) => Promise<{ messageId?: string }>; + sendSticker: ( + to: string, + stickerBuffer: Buffer, + options?: { mimetype?: string }, + ) => Promise<{ messageId?: string }>; + sendText: ( + to: string, + text: string, + options?: WhatsAppQaDriverSendTextOptions, + ) => Promise<{ messageId?: string }>; waitForMessage: (params: { match: (message: WhatsAppQaDriverObservedMessage) => boolean; + observedAfter?: Date; timeoutMs: number; }) => Promise; }; @@ -27,6 +121,8 @@ type MessageUpsertEvent = { messages?: WAMessage[]; }; +type ConnectionUpdateEvent = Partial; + type Waiter = { predicate: (message: WhatsAppQaDriverObservedMessage) => boolean; reject: (error: Error) => void; @@ -34,6 +130,148 @@ type Waiter = { timeout: NodeJS.Timeout; }; +type PendingNotificationsWaiter = { + reject: (error: Error) => void; + resolve: () => void; + timeout: NodeJS.Timeout; +}; + +function isRecord(value: unknown): value is Record { + return Boolean(value && typeof value === "object"); +} + +function readString(value: unknown): string | undefined { + return typeof value === "string" && value.trim() ? value.trim() : undefined; +} + +function readBoolean(value: unknown): boolean | undefined { + return typeof value === "boolean" ? value : undefined; +} + +function findMessageSection( + message: unknown, + sectionNames: readonly string[], +): Record | undefined { + if (!isRecord(message)) { + return undefined; + } + const queue: Array<{ depth: number; value: Record }> = [ + { depth: 0, value: message }, + ]; + const seen = new Set>(); + while (queue.length > 0) { + const current = queue.shift(); + if (!current || seen.has(current.value)) { + continue; + } + seen.add(current.value); + for (const sectionName of sectionNames) { + const section = current.value[sectionName]; + if (isRecord(section)) { + return section; + } + } + if (current.depth >= 4) { + continue; + } + for (const wrapperName of [ + "botInvokeMessage", + "documentWithCaptionMessage", + "ephemeralMessage", + "groupMentionedMessage", + "viewOnceMessage", + "viewOnceMessageV2", + "viewOnceMessageV2Extension", + ]) { + const wrapper = current.value[wrapperName]; + if (isRecord(wrapper) && isRecord(wrapper.message)) { + queue.push({ depth: current.depth + 1, value: wrapper.message }); + } + } + } + return undefined; +} + +function readReaction(message: unknown): WhatsAppQaDriverObservedReaction | undefined { + const reaction = findMessageSection(message, ["reactionMessage"]); + if (!reaction) { + return undefined; + } + const emoji = readString(reaction.text) ?? ""; + const key = isRecord(reaction.key) ? reaction.key : undefined; + return { + emoji, + fromMe: readBoolean(key?.fromMe), + messageId: readString(key?.id), + participant: readString(key?.participant), + }; +} + +function readPoll(message: unknown): WhatsAppQaDriverObservedPoll | undefined { + const poll = findMessageSection(message, [ + "pollCreationMessage", + "pollCreationMessageV2", + "pollCreationMessageV3", + ]); + if (!poll) { + return undefined; + } + const rawOptions = Array.isArray(poll.options) ? poll.options : []; + const options = rawOptions + .map((option) => (isRecord(option) ? readString(option.optionName) : undefined)) + .filter((option): option is string => Boolean(option)); + return { + options, + question: readString(poll.name), + }; +} + +function readMedia(message: unknown): + | { + fileName?: string; + mediaType?: string; + } + | undefined { + const normalizedMessage = isRecord(message) + ? normalizeMessageContent(message as proto.IMessage) + : undefined; + const mediaSections = [ + "imageMessage", + "videoMessage", + "audioMessage", + "documentMessage", + "stickerMessage", + ]; + for (const sectionName of mediaSections) { + const section = findMessageSection(normalizedMessage ?? message, [sectionName]); + if (!section) { + continue; + } + const mediaMessage = { [sectionName]: section } as proto.IMessage; + return { + fileName: readString(section.fileName), + mediaType: resolveInboundMediaMimetype(mediaMessage), + }; + } + return undefined; +} + +function readQuotedMessage(message: WAMessage): WhatsAppQaDriverQuotedMessage | undefined { + const contextInfo = extractContextInfo(message.message ?? undefined); + const replyContext = describeReplyContext(message.message as proto.IMessage | undefined); + if (!contextInfo && !replyContext) { + return undefined; + } + if (!contextInfo?.stanzaId && !contextInfo?.participant && !replyContext?.body) { + return undefined; + } + return { + messageId: replyContext?.id ?? contextInfo?.stanzaId ?? undefined, + participant: replyContext?.sender?.jid ?? contextInfo?.participant ?? undefined, + text: replyContext?.body, + }; +} + function normalizeObservedMessage( message: WAMessage, authDir: string, @@ -41,17 +279,42 @@ function normalizeObservedMessage( if (message.key.fromMe) { return null; } - const text = extractText(message.message ?? undefined); - if (!text) { + const extractedText = extractText(message.message ?? undefined); + const location = extractLocationData(message.message as proto.IMessage | undefined); + const locationText = location ? formatLocationText(location) : undefined; + const text = [extractedText, locationText].filter(Boolean).join("\n").trim() || undefined; + const reaction = readReaction(message.message); + const poll = readPoll(message.message); + const media = readMedia(message.message); + const quoted = readQuotedMessage(message); + const kind: WhatsAppQaDriverObservedMessageKind = reaction + ? "reaction" + : poll + ? "poll" + : media + ? "media" + : location + ? "location" + : text + ? "text" + : "unknown"; + if (!text && kind === "unknown") { return null; } const fromJid = message.key.remoteJid ?? undefined; return { fromJid, fromPhoneE164: fromJid ? jidToE164(fromJid, { authDir }) : null, + hasMedia: media ? true : undefined, + kind, + mediaFileName: media?.fileName, + mediaType: media?.mediaType, messageId: message.key.id ?? undefined, observedAt: new Date().toISOString(), - text, + poll, + quoted, + reaction, + text: text ?? "", }; } @@ -67,14 +330,26 @@ function closeSocket(sock: Awaited>) { } } +function createConnectionClosedError(update: ConnectionUpdateEvent) { + const reason = update.lastDisconnect?.error; + const status = getStatusCode(reason); + const details = reason ? `: ${formatError(reason)}` : ""; + const statusLabel = typeof status === "number" ? ` (status ${status})` : ""; + return new Error(`WhatsApp QA driver connection closed${statusLabel}${details}`); +} + export async function startWhatsAppQaDriverSession(params: { authDir: string; connectionTimeoutMs?: number; + waitForPendingNotifications?: boolean; }): Promise { const sock = await createWaSocket(false, false, { authDir: params.authDir }); const observedMessages: WhatsAppQaDriverObservedMessage[] = []; + const pendingNotificationsWaiters: PendingNotificationsWaiter[] = []; const waiters: Waiter[] = []; let closed = false; + let closedError: Error | undefined; + let receivedPendingNotifications = false; const removeWaiter = (waiter: Waiter) => { const index = waiters.indexOf(waiter); @@ -84,6 +359,25 @@ export async function startWhatsAppQaDriverSession(params: { clearTimeout(waiter.timeout); }; + const removePendingNotificationsWaiter = (waiter: PendingNotificationsWaiter) => { + const index = pendingNotificationsWaiters.indexOf(waiter); + if (index >= 0) { + pendingNotificationsWaiters.splice(index, 1); + } + clearTimeout(waiter.timeout); + }; + + const markPendingNotificationsReceived = () => { + if (receivedPendingNotifications) { + return; + } + receivedPendingNotifications = true; + for (const waiter of pendingNotificationsWaiters.slice()) { + removePendingNotificationsWaiter(waiter); + waiter.resolve(); + } + }; + const observe = (message: WhatsAppQaDriverObservedMessage) => { observedMessages.push(message); for (const waiter of waiters.slice()) { @@ -104,11 +398,24 @@ export async function startWhatsAppQaDriverSession(params: { } }; + const onConnectionUpdate = (event: ConnectionUpdateEvent) => { + if (event.receivedPendingNotifications === true) { + markPendingNotificationsReceived(); + } + if (event.connection === "close") { + closeSessionResources(createConnectionClosedError(event)); + } + }; + const removeMessageListener = () => { const evWithOff = sock.ev as unknown as { - off?: (event: string, listener: (event: MessageUpsertEvent) => void) => void; + off?: ( + event: string, + listener: ((event: ConnectionUpdateEvent) => void) | ((event: MessageUpsertEvent) => void), + ) => void; }; evWithOff.off?.("messages.upsert", onMessagesUpsert); + evWithOff.off?.("connection.update", onConnectionUpdate); }; const closeSessionResources = (waiterError?: Error) => { @@ -116,6 +423,13 @@ export async function startWhatsAppQaDriverSession(params: { return; } closed = true; + closedError = waiterError; + for (const waiter of pendingNotificationsWaiters.slice()) { + removePendingNotificationsWaiter(waiter); + if (waiterError) { + waiter.reject(waiterError); + } + } for (const waiter of waiters.slice()) { removeWaiter(waiter); if (waiterError) { @@ -127,8 +441,35 @@ export async function startWhatsAppQaDriverSession(params: { }; sock.ev.on("messages.upsert", onMessagesUpsert); + sock.ev.on("connection.update", onConnectionUpdate); try { await waitForWaConnection(sock, { timeoutMs: params.connectionTimeoutMs ?? 45_000 }); + if (params.waitForPendingNotifications) { + await new Promise((resolve, reject) => { + if (receivedPendingNotifications) { + resolve(); + return; + } + if (closed) { + reject(closedError ?? new Error("WhatsApp QA driver session closed")); + return; + } + const timeoutMs = params.connectionTimeoutMs ?? 45_000; + const waiter: PendingNotificationsWaiter = { + resolve, + reject, + timeout: setTimeout(() => { + removePendingNotificationsWaiter(waiter); + reject( + new Error( + `timed out after ${timeoutMs}ms waiting for WhatsApp QA driver pending notifications`, + ), + ); + }, timeoutMs), + }; + pendingNotificationsWaiters.push(waiter); + }); + } } catch (error) { closeSessionResources( error instanceof Error ? error : new Error("failed starting WhatsApp QA driver session"), @@ -139,6 +480,7 @@ export async function startWhatsAppQaDriverSession(params: { const sendApi = createWebSendApi({ sock, defaultAccountId: "qa-driver", + authDir: params.authDir, }); return { @@ -148,20 +490,69 @@ export async function startWhatsAppQaDriverSession(params: { getObservedMessages() { return [...observedMessages]; }, - async sendText(to, text) { - const result = await sendApi.sendMessage(to, text); + async sendContact(to, contact) { + const result = await sendApi.sendContact(to, contact); + return { + messageId: result.messageId, + }; + }, + async sendLocation(to, location) { + const result = await sendApi.sendLocation(to, location); + return { + messageId: result.messageId, + }; + }, + async sendMedia(to, text, mediaBuffer, mediaType, options) { + const result = await sendApi.sendMessage(to, text, mediaBuffer, mediaType, options); + return { + messageId: result.messageId, + }; + }, + async sendPoll(to, poll) { + const result = await sendApi.sendPoll(to, poll); + return { + messageId: result.messageId, + }; + }, + async sendReaction(chatJid, messageId, emoji, options) { + const result = await sendApi.sendReaction( + chatJid, + messageId, + emoji, + options.fromMe, + options.participant, + ); + return { + messageId: result.messageId, + }; + }, + async sendSticker(to, stickerBuffer, options) { + const result = await sendApi.sendSticker(to, stickerBuffer, options); + return { + messageId: result.messageId, + }; + }, + async sendText(to, text, options) { + const result = await sendApi.sendMessage(to, text, undefined, undefined, options); return { messageId: result.messageId, }; }, async waitForMessage(paramsLocal) { - const existing = observedMessages.find(paramsLocal.match); + const predicate = (message: WhatsAppQaDriverObservedMessage) => + (!paramsLocal.observedAfter || + new Date(message.observedAt).getTime() >= paramsLocal.observedAfter.getTime()) && + paramsLocal.match(message); + const existing = observedMessages.find(predicate); if (existing) { return existing; } + if (closed) { + throw closedError ?? new Error("WhatsApp QA driver session closed"); + } return await new Promise((resolve, reject) => { const waiter: Waiter = { - predicate: paramsLocal.match, + predicate, resolve, reject, timeout: setTimeout(() => {