diff --git a/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.test.ts b/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.test.ts index 10c7b16b9b10..c847e8c7288f 100644 --- a/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.test.ts @@ -2,7 +2,6 @@ import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; import { afterEach, describe, expect, it, vi } from "vitest"; import { discordQaScenarioSupport } from "./discord-live.runtime.js"; -import { resolveDiscordQaScenarioIds } from "./scenario-selection.js"; const { testing } = discordQaScenarioSupport; @@ -285,23 +284,6 @@ describe("discord live qa runtime", () => { expect(testing.computeDiscordRttMs("bad", "2026-04-22T12:00:00.875Z")).toBeUndefined(); }); - it("includes the Discord live scenarios", () => { - expect(testing.findScenario().map((scenario) => scenario.id)).toEqual( - resolveDiscordQaScenarioIds({}), - ); - expect( - testing.findScenario(["discord-status-reactions-tool-only"]).map((scenario) => scenario.id), - ).toEqual(["discord-status-reactions-tool-only"]); - expect(testing.findScenario(["discord-voice-autojoin"]).map((scenario) => scenario.id)).toEqual( - ["discord-voice-autojoin"], - ); - expect( - testing - .findScenario(["discord-thread-reply-filepath-attachment"]) - .map((scenario) => scenario.id), - ).toEqual(["discord-thread-reply-filepath-attachment"]); - }); - it("collects the status reaction sequence across timeline snapshots", () => { expect( testing.collectSeenReactionSequence( @@ -462,12 +444,6 @@ describe("discord live qa runtime", () => { } }); - it("fails when any requested Discord scenario id is unknown", () => { - expect(() => testing.findScenario(["discord-canary", "typo-scenario"])).toThrow( - "unknown QA scenario id(s): typo-scenario", - ); - }); - it("lists Discord application commands through the REST API", async () => { vi.stubGlobal( "fetch", diff --git a/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.ts b/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.ts index 98a96a1587fa..3fd1f00f5ba2 100644 --- a/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.ts +++ b/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.ts @@ -19,7 +19,6 @@ import { z } from "zod"; import { startQaGatewayChild } from "../../gateway-child.js"; import { isTruthyOptIn } from "../../mantis-options.runtime.js"; import { assertLiveScenarioReply as assertDiscordScenarioReply } from "../shared/live-scenario-reply.js"; -import { resolveDiscordQaScenarioIds } from "./scenario-selection.js"; type DiscordQaRuntimeEnv = { guildId: string; @@ -30,15 +29,7 @@ type DiscordQaRuntimeEnv = { voiceChannelId?: string; }; -type DiscordQaScenarioId = - | "discord-canary" - | "discord-mention-gating" - | "discord-native-help-command-registration" - | "discord-voice-autojoin" - | "discord-thread-reply-filepath-attachment" - | "discord-status-reactions-tool-only"; - -type DiscordQaScenarioRun = +export type DiscordQaScenarioRun = | { kind: "channel-message"; expectReply: boolean; @@ -65,13 +56,16 @@ type DiscordQaScenarioRun = replyContent: string; }; -type DiscordQaScenarioDefinition = { - id: DiscordQaScenarioId; - title: string; - timeoutMs: number; +export type DiscordQaScenarioImplementation = { buildRun: (sutApplicationId: string) => DiscordQaScenarioRun; }; +type DiscordQaScenarioMetadata = { + id: string; + timeoutMs: number; + title: string; +}; + type DiscordUser = { id: string; username?: string; @@ -196,7 +190,7 @@ type DiscordReactionSnapshot = { type DiscordStatusReactionTimeline = { expectedSequence: string[]; htmlPath?: string; - scenarioId: DiscordQaScenarioId; + scenarioId: string; scenarioTitle: string; screenshotPath?: string; screenshotWarning?: string; @@ -215,7 +209,7 @@ type DiscordThreadReplyAttachmentEvidence = { messageContent?: string; messageId?: string; parentMessageId?: string; - scenarioId: DiscordQaScenarioId; + scenarioId: string; scenarioTitle: string; screenshotPath?: string; screenshotWarning?: string; @@ -234,85 +228,70 @@ const DISCORD_QA_ENV_KEYS = [ "OPENCLAW_QA_DISCORD_SUT_APPLICATION_ID", ] as const; -const DISCORD_QA_SCENARIOS: DiscordQaScenarioDefinition[] = [ - { - id: "discord-canary", - title: "Discord canary echo", - timeoutMs: 45_000, - buildRun: (sutApplicationId) => { - const token = `DISCORD_QA_ECHO_${randomUUID().slice(0, 8).toUpperCase()}`; - return { - kind: "channel-message", - expectReply: true, - input: `<@${sutApplicationId}> reply with only this exact marker: ${token}`, - expectedTextIncludes: [token], - matchText: token, - }; - }, +export const discordQaCanaryScenario: DiscordQaScenarioImplementation = { + buildRun: (sutApplicationId) => { + const token = `DISCORD_QA_ECHO_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + kind: "channel-message", + expectReply: true, + input: `<@${sutApplicationId}> reply with only this exact marker: ${token}`, + expectedTextIncludes: [token], + matchText: token, + }; }, - { - id: "discord-mention-gating", - title: "Discord unmentioned message does not trigger", - timeoutMs: 8_000, - buildRun: () => { - const token = `DISCORD_QA_NOMENTION_${randomUUID().slice(0, 8).toUpperCase()}`; - return { - kind: "channel-message", - expectReply: false, - input: `reply with only this exact marker: ${token}`, - matchText: token, - }; - }, +}; + +export const discordQaMentionGatingScenario: DiscordQaScenarioImplementation = { + buildRun: () => { + const token = `DISCORD_QA_NOMENTION_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + kind: "channel-message", + expectReply: false, + input: `reply with only this exact marker: ${token}`, + matchText: token, + }; }, - { - id: "discord-native-help-command-registration", - title: "Discord native help command is registered", - timeoutMs: 45_000, - buildRun: () => ({ - kind: "application-command-registration", - expectedCommandNames: ["help"], - }), +}; + +export const discordQaNativeHelpCommandRegistrationScenario: DiscordQaScenarioImplementation = { + buildRun: () => ({ + kind: "application-command-registration", + expectedCommandNames: ["help"], + }), +}; + +export const discordQaVoiceAutojoinScenario: DiscordQaScenarioImplementation = { + buildRun: () => ({ + kind: "voice-autojoin", + }), +}; + +export const discordQaStatusReactionsToolOnlyScenario: DiscordQaScenarioImplementation = { + buildRun: () => { + const token = `DISCORD_QA_STATUS_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + kind: "status-reactions-tool-only", + input: [ + `Mantis status reaction QA marker ${token}.`, + "Think briefly, then reply with only this exact marker:", + token, + ].join(" "), + expectedSequence: ["👀", DEFAULT_EMOJIS.thinking, DEFAULT_EMOJIS.done], + }; }, - { - id: "discord-voice-autojoin", - title: "Discord voice auto-join connects", - timeoutMs: 60_000, - buildRun: () => ({ - kind: "voice-autojoin", - }), +}; + +export const discordQaThreadReplyFilepathAttachmentScenario: DiscordQaScenarioImplementation = { + buildRun: () => { + const token = `DISCORD_QA_THREAD_FILE_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + kind: "thread-reply-filepath-attachment", + input: `Mantis Discord thread attachment parent ${token}`, + replyContent: `Mantis thread attachment reply ${token}`, + expectedAttachmentFilename: "mantis-thread-report.md", + }; }, - { - id: "discord-status-reactions-tool-only", - title: "Discord explicit status reactions run in tool-only reply mode", - timeoutMs: 75_000, - buildRun: () => { - const token = `DISCORD_QA_STATUS_${randomUUID().slice(0, 8).toUpperCase()}`; - return { - kind: "status-reactions-tool-only", - input: [ - `Mantis status reaction QA marker ${token}.`, - "Think briefly, then reply with only this exact marker:", - token, - ].join(" "), - expectedSequence: ["👀", DEFAULT_EMOJIS.thinking, DEFAULT_EMOJIS.done], - }; - }, - }, - { - id: "discord-thread-reply-filepath-attachment", - title: "Discord thread reply preserves filePath attachment", - timeoutMs: 45_000, - buildRun: () => { - const token = `DISCORD_QA_THREAD_FILE_${randomUUID().slice(0, 8).toUpperCase()}`; - return { - kind: "thread-reply-filepath-attachment", - input: `Mantis Discord thread attachment parent ${token}`, - replyContent: `Mantis thread attachment reply ${token}`, - expectedAttachmentFilename: "mantis-thread-report.md", - }; - }, - }, -]; +}; const discordQaCredentialPayloadSchema = z.object({ guildId: z.string().trim().min(1), @@ -951,7 +930,7 @@ async function observeStatusReactionTimeline(params: { channelId: string; expectedSequence: string[]; messageId: string; - scenarioId: DiscordQaScenarioId; + scenarioId: string; scenarioTitle: string; timeoutMs: number; token: string; @@ -1117,7 +1096,7 @@ async function runDiscordThreadReplyFilePathAttachmentScenario(params: { driverBotId: string; outputDir: string; runtimeEnv: DiscordQaRuntimeEnv; - scenario: DiscordQaScenarioDefinition; + scenario: DiscordQaScenarioMetadata; scenarioRun: Extract; sutAccountId: string; sutBotId: string; @@ -1344,16 +1323,6 @@ function buildObservedMessagesArtifact(params: { }); } -function findScenario(ids?: string[]) { - const requestedIds = resolveDiscordQaScenarioIds({ scenarioIds: ids }); - const scenariosById = new Map(DISCORD_QA_SCENARIOS.map((scenario) => [scenario.id, scenario])); - const missingIds = requestedIds.filter((id) => !scenariosById.has(id as DiscordQaScenarioId)); - if (missingIds.length > 0) { - throw new Error(`unknown Discord QA scenario id(s): ${missingIds.join(", ")}`); - } - return requestedIds.map((id) => scenariosById.get(id as DiscordQaScenarioId)!); -} - function matchesDiscordScenarioReply(params: { channelId: string; message: DiscordObservedMessage; @@ -1401,7 +1370,6 @@ async function assertDiscordApplicationCommandsRegistered(params: { } const testing = { - DISCORD_QA_SCENARIOS, collectSeenReactionSequence, assertDiscordScenarioReply, assertDiscordApplicationCommandsRegistered, @@ -1409,7 +1377,6 @@ const testing = { buildDiscordWebMessageUrl, buildObservedMessagesArtifact, computeDiscordRttMs, - findScenario, getCurrentDiscordUser, observeStatusReactionTimeline, pollChannelMessages, diff --git a/extensions/qa-lab/src/live-transports/discord/scenario-environment.ts b/extensions/qa-lab/src/live-transports/discord/scenario-environment.ts index 57cfe170bc7e..d0b304a17ace 100644 --- a/extensions/qa-lab/src/live-transports/discord/scenario-environment.ts +++ b/extensions/qa-lab/src/live-transports/discord/scenario-environment.ts @@ -4,7 +4,11 @@ import { patchLiveQaGatewayConfig, readLiveQaGatewayConfig, } from "../shared/live-gateway-config.runtime.js"; -import { discordQaScenarioSupport } from "./discord-live.runtime.js"; +import { + discordQaScenarioSupport, + type DiscordQaScenarioImplementation, + type DiscordQaScenarioRun, +} from "./discord-live.runtime.js"; type AdapterFactory = NonNullable; type AdapterDefinition = Awaited>; @@ -18,18 +22,21 @@ type DiscordIdentity = Awaited< type DiscordObservedMessage = Parameters< typeof discordQaScenarioSupport.testing.pollChannelMessages >[0]["observedMessages"][number]; - export type DiscordQaScenarioEnvironment = { - cfg: OpenClawConfig; + configureScenario: (implementation: DiscordQaScenarioImplementation) => Promise<{ + cfg: OpenClawConfig; + run: DiscordQaScenarioRun; + voiceChannel?: Awaited< + ReturnType + >; + }>; driverIdentity: DiscordIdentity; observedMessages: DiscordObservedMessage[]; outputDir: string; runtimeEnv: DiscordRuntimeEnv; + scenario: { id: string; timeoutMs: number; title: string }; sutAccountId: string; sutIdentity: DiscordIdentity; - voiceChannel?: Awaited< - ReturnType - >; }; export function createDiscordQaScenarioEnvironment(params: { @@ -40,61 +47,64 @@ export function createDiscordQaScenarioEnvironment(params: { }) { const observedMessages: DiscordObservedMessage[] = []; const prepareFlow = async (input: FlowPreparationInput) => { - const scenarioId = input.config.discordScenarioId; - if (typeof scenarioId !== "string") { - throw new Error("Discord QA module flow requires config.discordScenarioId"); - } - const scenario = discordQaScenarioSupport.testing.findScenario([scenarioId])[0]; - if (!scenario) { - throw new Error(`unknown Discord QA scenario id: ${scenarioId}`); - } - const scenarioRun = scenario.buildRun(params.runtimeEnv.sutApplicationId); - const voiceChannel = - scenarioRun.kind === "voice-autojoin" - ? await discordQaScenarioSupport.testing.resolveDiscordQaVoiceChannel({ - guildId: params.runtimeEnv.guildId, - token: params.runtimeEnv.sutBotToken, - voiceChannelId: params.runtimeEnv.voiceChannelId, - }) - : undefined; - const snapshot = await readLiveQaGatewayConfig(input.gateway); - const cfg = discordQaScenarioSupport.testing.buildDiscordQaConfig( - snapshot.config as OpenClawConfig, - { - guildId: params.runtimeEnv.guildId, - channelId: params.runtimeEnv.channelId, - driverBotId: params.driverIdentity.id, - sutAccountId: params.accountId, - sutBotToken: params.runtimeEnv.sutBotToken, - }, - { - ...(voiceChannel - ? { voiceAutoJoin: { channelId: voiceChannel.id, guildId: params.runtimeEnv.guildId } } - : {}), - statusReactionsToolOnly: scenarioRun.kind === "status-reactions-tool-only", - }, - ); - await patchLiveQaGatewayConfig({ - gateway: input.gateway, - patch: cfg as Record, - replacePaths: ["channels.discord", "messages", "plugins"], - timeoutMs: input.timeoutMs, - waitForConfigRestartSettle: input.waitForConfigRestartSettle, - }); - await discordQaScenarioSupport.testing.waitForDiscordChannelRunning( - input.gateway as never, - params.accountId, - ); return { discordScenarioContext: { - cfg, + configureScenario: async (implementation: DiscordQaScenarioImplementation) => { + const run = implementation.buildRun(params.runtimeEnv.sutApplicationId); + const voiceChannel = + run.kind === "voice-autojoin" + ? await discordQaScenarioSupport.testing.resolveDiscordQaVoiceChannel({ + guildId: params.runtimeEnv.guildId, + token: params.runtimeEnv.sutBotToken, + voiceChannelId: params.runtimeEnv.voiceChannelId, + }) + : undefined; + const snapshot = await readLiveQaGatewayConfig(input.gateway); + const cfg = discordQaScenarioSupport.testing.buildDiscordQaConfig( + snapshot.config as OpenClawConfig, + { + guildId: params.runtimeEnv.guildId, + channelId: params.runtimeEnv.channelId, + driverBotId: params.driverIdentity.id, + sutAccountId: params.accountId, + sutBotToken: params.runtimeEnv.sutBotToken, + }, + { + ...(voiceChannel + ? { + voiceAutoJoin: { + channelId: voiceChannel.id, + guildId: params.runtimeEnv.guildId, + }, + } + : {}), + statusReactionsToolOnly: run.kind === "status-reactions-tool-only", + }, + ); + await patchLiveQaGatewayConfig({ + gateway: input.gateway, + patch: cfg as Record, + replacePaths: ["channels.discord", "messages", "plugins"], + timeoutMs: input.timeoutMs, + waitForConfigRestartSettle: input.waitForConfigRestartSettle, + }); + await discordQaScenarioSupport.testing.waitForDiscordChannelRunning( + input.gateway as never, + params.accountId, + ); + return { cfg, run, ...(voiceChannel ? { voiceChannel } : {}) }; + }, driverIdentity: params.driverIdentity, observedMessages, outputDir: input.outputDir, runtimeEnv: params.runtimeEnv, + scenario: { + id: input.scenarioId, + timeoutMs: input.timeoutMs, + title: input.scenarioTitle, + }, sutAccountId: params.accountId, sutIdentity: params.sutIdentity, - ...(voiceChannel ? { voiceChannel } : {}), } satisfies DiscordQaScenarioEnvironment, }; }; diff --git a/extensions/qa-lab/src/live-transports/discord/scenario-runtime.ts b/extensions/qa-lab/src/live-transports/discord/scenario-runtime.ts index b1df78a8dba0..791430c279d2 100644 --- a/extensions/qa-lab/src/live-transports/discord/scenario-runtime.ts +++ b/extensions/qa-lab/src/live-transports/discord/scenario-runtime.ts @@ -1,13 +1,25 @@ import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; -import { discordQaScenarioSupport } from "./discord-live.runtime.js"; +import { + discordQaScenarioSupport, + type DiscordQaScenarioImplementation, +} from "./discord-live.runtime.js"; import type { DiscordQaScenarioEnvironment } from "./scenario-environment.js"; -async function runDiscordScenario(environment: DiscordQaScenarioEnvironment, scenarioId: string) { - const scenario = discordQaScenarioSupport.testing.findScenario([scenarioId])[0]; - if (!scenario) { - throw new Error(`unknown Discord QA scenario id: ${scenarioId}`); - } - const run = scenario.buildRun(environment.runtimeEnv.sutApplicationId); +export { + discordQaCanaryScenario, + discordQaMentionGatingScenario, + discordQaNativeHelpCommandRegistrationScenario, + discordQaStatusReactionsToolOnlyScenario, + discordQaThreadReplyFilepathAttachmentScenario, + discordQaVoiceAutojoinScenario, +} from "./discord-live.runtime.js"; + +export async function runDiscordScenario( + environment: DiscordQaScenarioEnvironment, + implementation: DiscordQaScenarioImplementation, +) { + const scenario = environment.scenario; + const { cfg, run, voiceChannel } = await environment.configureScenario(implementation); if (run.kind === "application-command-registration") { const registered = await discordQaScenarioSupport.testing.assertDiscordApplicationCommandsRegistered({ @@ -19,13 +31,13 @@ async function runDiscordScenario(environment: DiscordQaScenarioEnvironment, sce return { details: `native command registered (${registered.commandNames.join(", ")})` }; } if (run.kind === "voice-autojoin") { - if (!environment.voiceChannel) { + if (!voiceChannel) { throw new Error("Discord voice auto-join scenario did not resolve a voice channel."); } await discordQaScenarioSupport.testing.waitForDiscordVoiceState({ token: environment.runtimeEnv.sutBotToken, guildId: environment.runtimeEnv.guildId, - channelId: environment.voiceChannel.id, + channelId: voiceChannel.id, sutBotId: environment.sutIdentity.id, timeoutMs: scenario.timeoutMs, }); @@ -34,7 +46,7 @@ async function runDiscordScenario(environment: DiscordQaScenarioEnvironment, sce if (run.kind === "thread-reply-filepath-attachment") { const result = await discordQaScenarioSupport.testing.runDiscordThreadReplyFilePathAttachmentScenario({ - cfg: environment.cfg, + cfg, driverBotId: environment.driverIdentity.id, outputDir: environment.outputDir, runtimeEnv: environment.runtimeEnv, @@ -116,18 +128,3 @@ async function runDiscordScenario(environment: DiscordQaScenarioEnvironment, sce throw error; } } - -export const runDiscordCanaryScenario = (context: DiscordQaScenarioEnvironment) => - runDiscordScenario(context, "discord-canary"); -export const runDiscordMentionGatingScenario = (context: DiscordQaScenarioEnvironment) => - runDiscordScenario(context, "discord-mention-gating"); -export const runDiscordNativeHelpCommandRegistrationScenario = ( - context: DiscordQaScenarioEnvironment, -) => runDiscordScenario(context, "discord-native-help-command-registration"); -export const runDiscordVoiceAutojoinScenario = (context: DiscordQaScenarioEnvironment) => - runDiscordScenario(context, "discord-voice-autojoin"); -export const runDiscordStatusReactionsToolOnlyScenario = (context: DiscordQaScenarioEnvironment) => - runDiscordScenario(context, "discord-status-reactions-tool-only"); -export const runDiscordThreadReplyFilepathAttachmentScenario = ( - context: DiscordQaScenarioEnvironment, -) => runDiscordScenario(context, "discord-thread-reply-filepath-attachment"); diff --git a/extensions/qa-lab/src/live-transports/matrix/scenarios/scenario-environment.test.ts b/extensions/qa-lab/src/live-transports/matrix/scenarios/scenario-environment.test.ts index 05962937a825..ebfa5dd35b5f 100644 --- a/extensions/qa-lab/src/live-transports/matrix/scenarios/scenario-environment.test.ts +++ b/extensions/qa-lab/src/live-transports/matrix/scenarios/scenario-environment.test.ts @@ -77,6 +77,8 @@ describe("matrix scenario environment", () => { config: {}, gateway, outputDir: "/tmp/matrix-qa/output", + scenarioId: "matrix-observer-reset", + scenarioTitle: "Matrix observer reset", timeoutMs: 1_000, waitForConfigRestartSettle: vi.fn(), }; @@ -168,6 +170,8 @@ describe("matrix scenario environment", () => { config: {}, gateway, outputDir: "/tmp/matrix-qa/output", + scenarioId: "matrix-approval", + scenarioTitle: "Matrix approval", timeoutMs: 1_000, waitForConfigRestartSettle, }); @@ -285,6 +289,8 @@ describe("matrix scenario environment", () => { config: {}, gateway, outputDir: "/tmp/matrix-qa/output", + scenarioId: "matrix-restart", + scenarioTitle: "Matrix restart", timeoutMs: 1_000, waitForConfigRestartSettle, }); diff --git a/extensions/qa-lab/src/live-transports/scenario-module-parity.test.ts b/extensions/qa-lab/src/live-transports/scenario-module-parity.test.ts index 94b93a3db541..58fdab90292a 100644 --- a/extensions/qa-lab/src/live-transports/scenario-module-parity.test.ts +++ b/extensions/qa-lab/src/live-transports/scenario-module-parity.test.ts @@ -7,25 +7,36 @@ import * as whatsappScenarioRuntime from "./whatsapp/scenario-runtime.js"; const LANES = [ { channel: "discord", + contextExpression: "discordScenarioContext", modulePath: "./live-transports/discord/scenario-runtime.js", + runnerName: "runDiscordScenario", runtime: discordScenarioRuntime, }, { channel: "slack", + contextExpression: "slackScenarioContext", modulePath: "./live-transports/slack/scenario-runtime.js", + runnerName: "runSlackScenario", runtime: slackScenarioRuntime, }, { channel: "whatsapp", + contextExpression: "whatsappScenarioContext", modulePath: "./live-transports/whatsapp/scenario-runtime.js", + runnerName: "runWhatsAppScenario", runtime: whatsappScenarioRuntime, }, ] as const; -function readScenarioModuleCallName( +type ScenarioModuleCall = { + args?: unknown[]; + call: string; +}; + +function readScenarioModuleCall( scenario: QaSeedScenarioWithSource, modulePath: string, -): string | undefined { +): ScenarioModuleCall | undefined { if (scenario.execution.kind !== "flow" || !scenario.execution.flow) { return undefined; } @@ -48,7 +59,7 @@ function readScenarioModuleCallName( } const callPrefix = "scenarioModule."; const callAction = actions.find( - (action): action is { call: string } => + (action): action is ScenarioModuleCall => typeof action === "object" && action !== null && "call" in action && @@ -58,27 +69,42 @@ function readScenarioModuleCallName( if (!callAction) { throw new Error(`scenario module flow has no call: ${scenario.id}`); } - return callAction.call.slice(callPrefix.length); + return { ...callAction, call: callAction.call.slice(callPrefix.length) }; } -describe("live transport scenario module parity", () => { +function readExpression(value: unknown): string | undefined { + return typeof value === "object" && + value !== null && + "expr" in value && + typeof value.expr === "string" + ? value.expr + : undefined; +} + +describe("live transport scenario module routing", () => { it.each(LANES)( - "keeps $channel scenario definitions and runtime exports in one-to-one parity", - ({ channel, modulePath, runtime }) => { + "routes every $channel flow through one shared channel runner", + ({ channel, contextExpression, modulePath, runnerName, runtime }) => { + expect(Reflect.get(runtime, runnerName)).toBeTypeOf("function"); + const bindings = readQaScenarioPack().scenarios.flatMap((scenario) => { if (scenario.execution.kind !== "flow" || scenario.execution.channel !== channel) { return []; } - const callName = readScenarioModuleCallName(scenario, modulePath); - return callName ? [{ callName, scenarioId: scenario.id }] : []; - }); - const callNames = bindings.map(({ callName, scenarioId }) => { - expect(Reflect.get(runtime, callName), scenarioId).toBeTypeOf("function"); - return callName; + const call = readScenarioModuleCall(scenario, modulePath); + return call ? [{ call, scenarioId: scenario.id }] : []; }); - expect(new Set(callNames).size).toBe(callNames.length); - expect(callNames.toSorted()).toEqual(Object.keys(runtime).toSorted()); + for (const { call, scenarioId } of bindings) { + expect(call.call, scenarioId).toBe(runnerName); + expect(call.args, scenarioId).toHaveLength(2); + expect(readExpression(call.args?.[0]), scenarioId).toBe(contextExpression); + + const implementationExpression = readExpression(call.args?.[1]); + const match = implementationExpression?.match(/^scenarioModule\["(\w+)"\]$/); + expect(match, scenarioId).toBeTruthy(); + expect(Reflect.get(runtime, match?.[1] ?? ""), scenarioId).toBeTypeOf("object"); + } }, ); }); diff --git a/extensions/qa-lab/src/live-transports/slack/scenario-environment.test.ts b/extensions/qa-lab/src/live-transports/slack/scenario-environment.test.ts index f091c5a69ba6..92b8c13e3738 100644 --- a/extensions/qa-lab/src/live-transports/slack/scenario-environment.test.ts +++ b/extensions/qa-lab/src/live-transports/slack/scenario-environment.test.ts @@ -1,5 +1,7 @@ +// QA Lab Slack tests cover module-specific flow preparation boundaries. import { describe, expect, it, vi } from "vitest"; import { createSlackQaScenarioEnvironment } from "./scenario-environment.js"; +import { slackQaAllowlistBlockScenario } from "./slack-live.scenario-implementations.js"; function createEnvironment() { return createSlackQaScenarioEnvironment({ @@ -15,21 +17,26 @@ function createEnvironment() { }); } -describe("Slack live scenario environment", () => { - it("leaves generic declarative flows to their own config preparation", async () => { +describe("Slack scenario environment", () => { + it("leaves generic declarative flows on the adapter's baseline config", async () => { const gatewayCall = vi.fn(); const { prepareFlow } = createEnvironment(); - await expect( - prepareFlow({ - config: { policyKey: "dmPolicy", policyValue: "disabled" }, - gateway: { call: gatewayCall } as never, - outputDir: "/tmp/slack-output", - primaryModel: "mock-openai/gpt-5.6-luna", - timeoutMs: 60_000, - waitForConfigRestartSettle: vi.fn(), - }), - ).resolves.toBeUndefined(); + const prepared = await prepareFlow({ + config: { replyMarker: "QA-THREAD-FOLLOW-UP-OK" }, + gateway: { call: gatewayCall } as never, + outputDir: "/tmp/slack-output", + primaryModel: "mock-openai/gpt-5.6-luna", + scenarioId: "thread-follow-up", + scenarioTitle: "Thread follow-up", + timeoutMs: 60_000, + waitForConfigRestartSettle: vi.fn(), + }); + expect(prepared.slackScenarioContext.scenario).toEqual({ + id: "thread-follow-up", + timeoutMs: 60_000, + title: "Thread follow-up", + }); expect(gatewayCall).not.toHaveBeenCalled(); }); @@ -59,16 +66,19 @@ describe("Slack live scenario environment", () => { throw new Error(`unexpected gateway method: ${method}`); }); const { prepareFlow } = createEnvironment(); - - await prepareFlow({ - config: { slackScenarioId: "slack-allowlist-block" }, + const prepared = await prepareFlow({ + config: {}, gateway: { call: gatewayCall } as never, outputDir: "/tmp/slack-output", primaryModel: "mock-openai/gpt-5.6-luna", + scenarioId: "slack-allowlist-block", + scenarioTitle: "Slack allowlist block", timeoutMs: 60_000, waitForConfigRestartSettle: vi.fn(), }); + await prepared.slackScenarioContext.configureScenario(slackQaAllowlistBlockScenario); + const patchCall = gatewayCall.mock.calls.find(([method]) => method === "config.patch"); if (!patchCall) { throw new Error("config.patch was not called"); diff --git a/extensions/qa-lab/src/live-transports/slack/scenario-environment.ts b/extensions/qa-lab/src/live-transports/slack/scenario-environment.ts index 213672bf7f8a..33e68c653a71 100644 --- a/extensions/qa-lab/src/live-transports/slack/scenario-environment.ts +++ b/extensions/qa-lab/src/live-transports/slack/scenario-environment.ts @@ -10,25 +10,31 @@ import { buildSlackQaConfig } from "./slack-live.config.js"; import type { SlackAuthIdentity, SlackObservedMessage, + SlackQaScenarioImplementation, SlackQaScenarioContext, + SlackQaScenarioMetadata, + SlackQaScenarioRun, } from "./slack-live.contracts.js"; import { assertSlackCodexApprovalModelSupported } from "./slack-live.contracts.js"; import { waitForSlackChannelStable } from "./slack-live.message-observations.js"; import { sendSlackChannelMessage } from "./slack-live.observations.js"; -import { getSlackQaScenarioDefinition } from "./slack-live.scenarios.js"; type AdapterFactory = NonNullable; type AdapterDefinition = Awaited>; type FlowPreparationInput = Parameters>[0]; export type SlackQaScenarioEnvironment = { - cfg: OpenClawConfig; channelId: string; + configureScenario: (implementation: SlackQaScenarioImplementation) => Promise<{ + cfg: OpenClawConfig; + primaryModel: string; + run: SlackQaScenarioRun; + }>; context: Omit; gatewayDebugDirPath: string; observedMessages: SlackObservedMessage[]; outputDir: string; - primaryModel: string; + scenario: SlackQaScenarioMetadata; stopGateway: (preserveDebugArtifacts: boolean) => Promise; sutAccountId: string; sutIdentity: SlackAuthIdentity; @@ -62,41 +68,6 @@ export function createSlackQaScenarioEnvironment(params: { const observedMessages: SlackObservedMessage[] = []; const prepareFlow = async (input: FlowPreparationInput) => { - const scenarioId = input.config.slackScenarioId; - if (typeof scenarioId !== "string") { - return undefined; - } - if (!input.primaryModel) { - throw new Error("Slack QA module flow requires a primary model"); - } - const primaryModel = input.primaryModel; - const scenario = getSlackQaScenarioDefinition(scenarioId); - const scenarioRun = scenario.buildRun(params.sutIdentity.userId); - if (scenarioRun.kind === "codex-approval") { - assertSlackCodexApprovalModelSupported(primaryModel); - } - const snapshot = await readLiveQaGatewayConfig(input.gateway); - const cfg = buildSlackQaConfig(snapshot.config as OpenClawConfig, { - channelId: params.channelId, - driverBotUserId: params.driverBotUserId, - overrides: scenario.configOverrides, - primaryModel, - sutAccountId: params.accountId, - sutAppToken: params.sutAppToken, - sutBotToken: params.sutBotToken, - }); - await patchLiveQaGatewayConfig({ - gateway: input.gateway, - patch: cfg as Record, - replacePaths: resolveSlackQaReplacePaths(params.accountId, params.channelId), - timeoutMs: input.timeoutMs, - waitForConfigRestartSettle: input.waitForConfigRestartSettle, - }); - const readinessMode = - scenarioRun.kind === "approval" || scenarioRun.kind === "codex-approval" - ? "started" - : "connected"; - await waitForSlackChannelStable(input.gateway as never, params.accountId, readinessMode); const context = { channelId: params.channelId, driverClient: params.driverClient, @@ -115,13 +86,47 @@ export function createSlackQaScenarioEnvironment(params: { } satisfies Omit; return { slackScenarioContext: { - cfg, channelId: params.channelId, + configureScenario: async (implementation: SlackQaScenarioImplementation) => { + if (!input.primaryModel) { + throw new Error("Slack QA module flow requires a primary model"); + } + const primaryModel = input.primaryModel; + const run = implementation.buildRun(params.sutIdentity.userId); + if (run.kind === "codex-approval") { + assertSlackCodexApprovalModelSupported(primaryModel); + } + const snapshot = await readLiveQaGatewayConfig(input.gateway); + const cfg = buildSlackQaConfig(snapshot.config as OpenClawConfig, { + channelId: params.channelId, + driverBotUserId: params.driverBotUserId, + overrides: implementation.configOverrides, + primaryModel, + sutAccountId: params.accountId, + sutAppToken: params.sutAppToken, + sutBotToken: params.sutBotToken, + }); + await patchLiveQaGatewayConfig({ + gateway: input.gateway, + patch: cfg as Record, + replacePaths: resolveSlackQaReplacePaths(params.accountId, params.channelId), + timeoutMs: input.timeoutMs, + waitForConfigRestartSettle: input.waitForConfigRestartSettle, + }); + const readinessMode = + run.kind === "approval" || run.kind === "codex-approval" ? "started" : "connected"; + await waitForSlackChannelStable(input.gateway as never, params.accountId, readinessMode); + return { cfg, primaryModel, run }; + }, context, gatewayDebugDirPath: path.join(input.outputDir, "gateway-debug"), observedMessages, outputDir: input.outputDir, - primaryModel, + scenario: { + id: input.scenarioId, + timeoutMs: input.timeoutMs, + title: input.scenarioTitle, + }, stopGateway: async (preserveDebugArtifacts: boolean) => { if (!input.gateway.stop) { throw new Error("Slack QA scenario requires gateway stop support"); diff --git a/extensions/qa-lab/src/live-transports/slack/scenario-runtime.ts b/extensions/qa-lab/src/live-transports/slack/scenario-runtime.ts index 34c8b14f278e..7616f7ff64ca 100644 --- a/extensions/qa-lab/src/live-transports/slack/scenario-runtime.ts +++ b/extensions/qa-lab/src/live-transports/slack/scenario-runtime.ts @@ -1,7 +1,10 @@ import type { SlackQaScenarioEnvironment } from "./scenario-environment.js"; import { runSlackApprovalScenario } from "./slack-live.approvals.js"; import { runSlackCodexApprovalScenario } from "./slack-live.codex-approval-runner.js"; -import type { SlackQaMessageScenarioRun } from "./slack-live.contracts.js"; +import type { + SlackQaMessageScenarioRun, + SlackQaScenarioImplementation, +} from "./slack-live.contracts.js"; import { observeSlackScenarioMessages, waitForSlackNoReply, @@ -12,7 +15,27 @@ import { collectSlackBlockText, sendSlackChannelMessage, } from "./slack-live.observations.js"; -import { getSlackQaScenarioDefinition } from "./slack-live.scenarios.js"; + +export { + slackQaAllowlistBlockScenario, + slackQaApprovalExecNativeScenario, + slackQaApprovalPluginNativeScenario, + slackQaCanaryScenario, + slackQaChannelDisabledWarningScenario, + slackQaChartPresentationNativeScenario, + slackQaCodexApprovalExecNativeScenario, + slackQaCodexApprovalPluginNativeScenario, + slackQaMentionGatingScenario, + slackQaMpimAppMentionDedupeScenario, + slackQaProgressCommentaryFalseScenario, + slackQaProgressCommentaryOmittedScenario, + slackQaProgressCommentaryTrueScenario, + slackQaProgressCommentaryVerboseDedupeScenario, + slackQaReactionGlyphNativeScenario, + slackQaTableInvalidBlocksFallbackScenario, + slackQaTablePresentationNativeScenario, + slackQaTopLevelReplyShapeScenario, +} from "./slack-live.scenario-implementations.js"; async function runSlackMessageScenario(params: { environment: SlackQaScenarioEnvironment; @@ -108,12 +131,15 @@ async function runSlackMessageScenario(params: { } } -async function runSlackScenario(environment: SlackQaScenarioEnvironment, scenarioId: string) { - const scenario = getSlackQaScenarioDefinition(scenarioId); - const run = scenario.buildRun(environment.sutIdentity.userId); +export async function runSlackScenario( + environment: SlackQaScenarioEnvironment, + implementation: SlackQaScenarioImplementation, +) { + const scenario = environment.scenario; + const { cfg, primaryModel, run } = await environment.configureScenario(implementation); if (run.kind === "direct-transport") { const result = await run.execute({ - cfg: environment.cfg, + cfg, channelId: environment.channelId, sutAccountId: environment.sutAccountId, sutIdentity: environment.sutIdentity, @@ -159,7 +185,7 @@ async function runSlackScenario(environment: SlackQaScenarioEnvironment, scenari channelId: environment.channelId, context: environment.context, observedMessages: environment.observedMessages, - primaryModel: environment.primaryModel, + primaryModel, run, scenario, stopGateway: environment.stopGateway, @@ -178,41 +204,3 @@ async function runSlackScenario(environment: SlackQaScenarioEnvironment, scenari timeoutMs: scenario.timeoutMs, }); } - -export const runSlackCanaryScenario = (context: SlackQaScenarioEnvironment) => - runSlackScenario(context, "slack-canary"); -export const runSlackMentionGatingScenario = (context: SlackQaScenarioEnvironment) => - runSlackScenario(context, "slack-mention-gating"); -export const runSlackMpimAppMentionDedupeScenario = (context: SlackQaScenarioEnvironment) => - runSlackScenario(context, "slack-mpim-app-mention-dedupe"); -export const runSlackAllowlistBlockScenario = (context: SlackQaScenarioEnvironment) => - runSlackScenario(context, "slack-allowlist-block"); -export const runSlackChannelDisabledWarningScenario = (context: SlackQaScenarioEnvironment) => - runSlackScenario(context, "slack-channel-disabled-warning"); -export const runSlackTopLevelReplyShapeScenario = (context: SlackQaScenarioEnvironment) => - runSlackScenario(context, "slack-top-level-reply-shape"); -export const runSlackProgressCommentaryTrueScenario = (context: SlackQaScenarioEnvironment) => - runSlackScenario(context, "slack-progress-commentary-true"); -export const runSlackProgressCommentaryFalseScenario = (context: SlackQaScenarioEnvironment) => - runSlackScenario(context, "slack-progress-commentary-false"); -export const runSlackProgressCommentaryOmittedScenario = (context: SlackQaScenarioEnvironment) => - runSlackScenario(context, "slack-progress-commentary-omitted"); -export const runSlackProgressCommentaryVerboseDedupeScenario = ( - context: SlackQaScenarioEnvironment, -) => runSlackScenario(context, "slack-progress-commentary-verbose-dedupe"); -export const runSlackChartPresentationNativeScenario = (context: SlackQaScenarioEnvironment) => - runSlackScenario(context, "slack-chart-presentation-native"); -export const runSlackTablePresentationNativeScenario = (context: SlackQaScenarioEnvironment) => - runSlackScenario(context, "slack-table-presentation-native"); -export const runSlackTableInvalidBlocksFallbackScenario = (context: SlackQaScenarioEnvironment) => - runSlackScenario(context, "slack-table-invalid-blocks-fallback"); -export const runSlackReactionGlyphNativeScenario = (context: SlackQaScenarioEnvironment) => - runSlackScenario(context, "slack-reaction-glyph-native"); -export const runSlackApprovalExecNativeScenario = (context: SlackQaScenarioEnvironment) => - runSlackScenario(context, "slack-approval-exec-native"); -export const runSlackApprovalPluginNativeScenario = (context: SlackQaScenarioEnvironment) => - runSlackScenario(context, "slack-approval-plugin-native"); -export const runSlackCodexApprovalExecNativeScenario = (context: SlackQaScenarioEnvironment) => - runSlackScenario(context, "slack-codex-approval-exec-native"); -export const runSlackCodexApprovalPluginNativeScenario = (context: SlackQaScenarioEnvironment) => - runSlackScenario(context, "slack-codex-approval-plugin-native"); diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.approval-checkpoint.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.approval-checkpoint.ts index a4d0c9f13229..4e9afe2c207e 100644 --- a/extensions/qa-lab/src/live-transports/slack/slack-live.approval-checkpoint.ts +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.approval-checkpoint.ts @@ -9,7 +9,6 @@ import { import { SLACK_QA_APPROVAL_DECISION_TIMEOUT_MS, SLACK_QA_APPROVAL_CHECKPOINT_DEFAULT_TIMEOUT_MS, - type SlackQaScenarioId, type SlackQaApprovalKind, type SlackQaApprovalDecision, type SlackQaApprovalScenarioRun, @@ -79,7 +78,7 @@ export async function writeSlackApprovalCheckpoint(params: { decision?: SlackQaApprovalDecision; message: SlackMessage; observedAt: string; - scenarioId: SlackQaScenarioId; + scenarioId: string; state: SlackApprovalCheckpointState; }) { const config = resolveSlackApprovalCheckpointConfig(); diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.approvals.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.approvals.ts index b0980a8403eb..e4b2d6bc3226 100644 --- a/extensions/qa-lab/src/live-transports/slack/slack-live.approvals.ts +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.approvals.ts @@ -13,7 +13,7 @@ import { type SlackQaApprovalDecision, type SlackQaApprovalScenarioRun, type SlackQaScenarioContext, - type SlackQaScenarioDefinition, + type SlackQaScenarioMetadata, type SlackAuthIdentity, type SlackObservedMessage, type SlackApprovalArtifact, @@ -291,7 +291,7 @@ export async function runSlackApprovalScenario(params: { context: Omit; observedMessages: SlackObservedMessage[]; run: SlackQaApprovalScenarioRun; - scenario: SlackQaScenarioDefinition; + scenario: SlackQaScenarioMetadata; sutAccountId: string; }) { const requestStartedAt = new Date(); diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.codex-approval-runner.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.codex-approval-runner.ts index 678899881318..b196de3207c5 100644 --- a/extensions/qa-lab/src/live-transports/slack/slack-live.codex-approval-runner.ts +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.codex-approval-runner.ts @@ -20,7 +20,7 @@ import { import type { SlackQaCodexApprovalScenarioRun, SlackQaScenarioContext, - SlackQaScenarioDefinition, + SlackQaScenarioMetadata, SlackObservedMessage, SlackApprovalArtifact, } from "./slack-live.contracts.js"; @@ -31,7 +31,7 @@ export async function runSlackCodexApprovalScenario(params: { observedMessages: SlackObservedMessage[]; primaryModel: string; run: SlackQaCodexApprovalScenarioRun; - scenario: SlackQaScenarioDefinition; + scenario: SlackQaScenarioMetadata; stopGateway: (preserveDebugArtifacts: boolean) => Promise; sutAccountId: string; }) { @@ -97,7 +97,7 @@ async function runSlackCodexApprovalScenarioInner(params: { observedMessages: SlackObservedMessage[]; primaryModel: string; run: SlackQaCodexApprovalScenarioRun; - scenario: SlackQaScenarioDefinition; + scenario: SlackQaScenarioMetadata; sutAccountId: string; }) { const requestStartedAt = new Date(); diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.codex-approval.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.codex-approval.ts index f559b716dc7a..cd67a394a18e 100644 --- a/extensions/qa-lab/src/live-transports/slack/slack-live.codex-approval.ts +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.codex-approval.ts @@ -12,7 +12,7 @@ import { type SlackQaCodexApprovalMethod, type SlackQaCodexApprovalScenarioRun, type SlackQaScenarioContext, - type SlackQaScenarioDefinition, + type SlackQaScenarioMetadata, } from "./slack-live.contracts.js"; export function resolveCodexFileApprovalTargetPath(token: string) { @@ -224,7 +224,7 @@ export async function startCodexApprovalAgentRun(params: { primaryModel: string; run: SlackQaCodexApprovalScenarioRun; runId: string; - scenario: SlackQaScenarioDefinition; + scenario: SlackQaScenarioMetadata; sessionKey: string; sutAccountId: string; }) { @@ -258,7 +258,7 @@ export async function startCodexApprovalAgentRun(params: { } export function buildCodexApprovalSessionKey(params: { - scenario: SlackQaScenarioDefinition; + scenario: SlackQaScenarioMetadata; token: string; }) { return `agent:qa:${params.scenario.id}-${params.token.toLowerCase()}`; diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.contracts.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.contracts.ts index 2e7cd23b02c1..8dfc056fbf5c 100644 --- a/extensions/qa-lab/src/live-transports/slack/slack-live.contracts.ts +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.contracts.ts @@ -4,7 +4,6 @@ import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; import { z } from "zod"; import type { startQaGatewayChild } from "../../gateway-child.js"; import { splitQaModelRef } from "../../model-selection.js"; -import type { RuntimeId } from "../../runtime-parity.js"; export type SlackQaRuntimeEnv = { channelId: string; @@ -85,26 +84,6 @@ export const SLACK_QA_NATIVE_TABLE = { // These scenarios force the Codex harness, whose default provider set is intentionally narrow. const SLACK_QA_CODEX_PROVIDER_IDS = new Set(["codex", "openai"]); -export type SlackQaScenarioId = - | "slack-allowlist-block" - | "slack-approval-exec-native" - | "slack-approval-plugin-native" - | "slack-canary" - | "slack-codex-approval-exec-native" - | "slack-codex-approval-plugin-native" - | "slack-chart-presentation-native" - | "slack-channel-disabled-warning" - | "slack-mention-gating" - | "slack-mpim-app-mention-dedupe" - | "slack-progress-commentary-false" - | "slack-progress-commentary-omitted" - | "slack-progress-commentary-true" - | "slack-progress-commentary-verbose-dedupe" - | "slack-reaction-glyph-native" - | "slack-table-invalid-blocks-fallback" - | "slack-table-presentation-native" - | "slack-top-level-reply-shape"; - export type SlackQaApprovalKind = "exec" | "plugin"; export type SlackQaApprovalDecision = "allow-always" | "allow-once" | "deny"; export const SLACK_QA_APPROVAL_ACTION_PREFIX = "openclaw:approval:v1:"; @@ -184,7 +163,7 @@ export type SlackQaCodexApprovalScenarioRun = { token: string; }; -type SlackQaScenarioRun = +export type SlackQaScenarioRun = | SlackQaApprovalScenarioRun | SlackQaCodexApprovalScenarioRun | SlackQaDirectTransportScenarioRun @@ -230,13 +209,15 @@ export type SlackQaScenarioContext = { waitForReady: () => Promise; }; -export type SlackQaScenarioDefinition = { - id: SlackQaScenarioId; - title: string; - timeoutMs: number; +export type SlackQaScenarioImplementation = { buildRun: (sutUserId: string) => SlackQaScenarioRun; configOverrides?: SlackQaConfigOverrides; - forcedRuntime?: RuntimeId; +}; + +export type SlackQaScenarioMetadata = { + id: string; + timeoutMs: number; + title: string; }; export type SlackAuthIdentity = { diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts index 7e1a0276bc6a..b99a85f9d9ee 100644 --- a/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts @@ -1,5 +1,7 @@ // Qa Lab tests cover slack live plugin behavior. import { beforeEach, describe, expect, it, vi } from "vitest"; +import { readQaScenarioById } from "../../scenario-catalog.js"; +import { requireFlowScenario } from "../../scenario-catalog.test-utils.js"; import { testing as adapterTesting } from "./adapter.runtime.js"; import { resolveSlackQaScenarioIds } from "./scenario-selection.js"; import { resolveApprovalDecision } from "./slack-live.approvals.js"; @@ -13,7 +15,10 @@ import { parseSlackQaCredentialPayload, resolveSlackQaRuntimeEnv, } from "./slack-live.config.js"; -import { assertSlackCodexApprovalModelSupported } from "./slack-live.contracts.js"; +import { + assertSlackCodexApprovalModelSupported, + type SlackQaScenarioImplementation, +} from "./slack-live.contracts.js"; import { buildSlackInvalidBlocksTableProbe } from "./slack-live.invalid-blocks.js"; import { observeSlackScenarioMessages, @@ -25,16 +30,32 @@ import { extractSlackNativeApprovalId, runSlackTableInvalidBlocksFallbackScenario, } from "./slack-live.observations.js"; -import { - getSlackQaScenarioDefinition, - listSlackQaScenarioCatalog, -} from "./slack-live.scenarios.js"; +import * as slackScenarioImplementations from "./slack-live.scenario-implementations.js"; + +function toSlackScenarioExportName(id: string): string { + const suffix = id + .replace(/^slack-/, "") + .split("-") + .map((part) => `${part.slice(0, 1).toUpperCase()}${part.slice(1)}`) + .join(""); + return `slackQa${suffix}Scenario`; +} function findScenario(ids?: string[]) { - const requestedIds = new Set(ids?.length ? ids : resolveSlackQaScenarioIds({})); - return listSlackQaScenarioCatalog() - .filter(({ id }) => requestedIds.has(id)) - .map(({ id }) => getSlackQaScenarioDefinition(id)); + return resolveSlackQaScenarioIds({ scenarioIds: ids }).map((id) => { + const implementation = ( + slackScenarioImplementations as unknown as Record + )[toSlackScenarioExportName(id)]; + if (!implementation) { + throw new Error(`missing Slack test implementation for ${id}`); + } + const scenario = requireFlowScenario(readQaScenarioById(id)); + return Object.assign({}, implementation, { + id, + timeoutMs: scenario.execution.timeoutMs ?? 60_000, + title: scenario.title, + }); + }); } const testing = { @@ -207,43 +228,29 @@ describe("Slack live QA runtime helpers", () => { }); it("selects native scenarios by explicit id", () => { - expect( - testing - .findScenario([ - "slack-chart-presentation-native", - "slack-table-presentation-native", - "slack-table-invalid-blocks-fallback", - "slack-progress-commentary-true", - "slack-progress-commentary-false", - "slack-progress-commentary-omitted", - "slack-progress-commentary-verbose-dedupe", - "slack-reaction-glyph-native", - "slack-approval-exec-native", - "slack-approval-plugin-native", - "slack-codex-approval-exec-native", - "slack-codex-approval-plugin-native", - "slack-channel-disabled-warning", - ]) - .map((scenario) => scenario.id), - ).toEqual([ - "slack-channel-disabled-warning", + const scenarioIds = [ + "slack-chart-presentation-native", + "slack-table-presentation-native", + "slack-table-invalid-blocks-fallback", "slack-progress-commentary-true", "slack-progress-commentary-false", "slack-progress-commentary-omitted", "slack-progress-commentary-verbose-dedupe", - "slack-chart-presentation-native", - "slack-table-presentation-native", - "slack-table-invalid-blocks-fallback", "slack-reaction-glyph-native", "slack-approval-exec-native", "slack-approval-plugin-native", "slack-codex-approval-exec-native", "slack-codex-approval-plugin-native", - ]); - expect(testing.findScenario(["slack-codex-approval-exec-native"])[0]?.forcedRuntime).toBe( - "codex", - ); - expect(testing.findScenario(["slack-canary"])[0]?.forcedRuntime).toBeUndefined(); + "slack-channel-disabled-warning", + ]; + const selectedIds = testing.findScenario(scenarioIds).map((scenario) => scenario.id); + expect(new Set(selectedIds)).toEqual(new Set(scenarioIds)); + expect( + requireFlowScenario(readQaScenarioById("slack-codex-approval-exec-native")).execution.runtime, + ).toBe("codex"); + expect( + requireFlowScenario(readQaScenarioById("slack-canary")).execution.runtime, + ).toBeUndefined(); }); it("accepts only Codex harness providers for Codex approval scenarios", () => { diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.scenario-implementations.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.scenario-implementations.ts new file mode 100644 index 000000000000..4c9a1db711ff --- /dev/null +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.scenario-implementations.ts @@ -0,0 +1,417 @@ +// QA Lab Slack live scenario implementations. +import { randomUUID } from "node:crypto"; +import { setTimeout as sleep } from "node:timers/promises"; +import { waitForSlackReaction } from "./slack-live.codex-approval.js"; +import { + SLACK_QA_REACTION_VERIFY_TIMEOUT_MS, + SLACK_QA_NATIVE_DATA_VERIFY_TIMEOUT_MS, + SLACK_QA_LOG_TAIL_TIMEOUT_MS, + type SlackQaScenarioImplementation, + type SlackQaScenarioContext, +} from "./slack-live.contracts.js"; +import { + isExpectedSlackNativeChartMessage, + isExpectedSlackNativeTableMessage, + runSlackTableInvalidBlocksFallbackScenario, + waitForSlackStoredMessage, +} from "./slack-live.observations.js"; +import { + buildSlackChartMessageToolArgs, + renderSlackChartAccessibleText, + buildSlackTableMessageToolArgs, + renderSlackTableAccessibleText, + buildSlackProgressCommentaryRun, +} from "./slack-live.scenario-fixtures.js"; + +export const slackQaCanaryScenario: SlackQaScenarioImplementation = { + buildRun: (sutUserId) => { + const token = `SLACK_QA_ECHO_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + expectReply: true, + input: `<@${sutUserId}> reply with only this exact marker: ${token}`, + matchText: token, + }; + }, +}; + +export const slackQaMentionGatingScenario: SlackQaScenarioImplementation = { + buildRun: () => { + const token = `SLACK_QA_NOMENTION_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + expectReply: false, + input: `reply with only this exact marker: ${token}`, + matchText: token, + }; + }, +}; + +export const slackQaMpimAppMentionDedupeScenario: SlackQaScenarioImplementation = { + configOverrides: { groupDmEnabled: true }, + buildRun: (sutUserId) => { + const token = `SLACK_QA_MPIM_${randomUUID().slice(0, 8).toUpperCase()}`; + let openedChannelId: string | undefined; + const closeOpenedChannel = async (context: Omit) => { + if (!openedChannelId) { + return; + } + const channelId = openedChannelId; + for (let attempt = 1; attempt <= 2; attempt += 1) { + try { + await context.sutReadClient.conversations.close({ channel: channelId }); + openedChannelId = undefined; + return; + } catch (error) { + if (attempt === 2) { + throw error; + } + // Retain ownership until Slack confirms closure; one bounded retry + // covers a transient API failure without hiding a leaked MPIM. + await sleep(500); + } + } + }; + return { + expectReply: true, + input: `<@${sutUserId}> reply with only this exact marker: ${token}`, + matchText: token, + settleObservedMs: 60_000, + beforeRun: async (context) => { + const driverAuth = await context.driverClient.auth.test(); + const driverUserId = driverAuth.user_id?.trim(); + if (!driverUserId) { + throw new Error("Slack QA driver auth.test returned no user_id"); + } + const members = await context.sutReadClient.conversations.members({ + channel: context.channelId, + limit: 100, + }); + const candidateUserIds = (members.members ?? []).filter( + (userId) => userId !== driverUserId && userId !== context.sutIdentity.userId, + ); + for (const userId of candidateUserIds) { + const user = (await context.sutReadClient.users.info({ user: userId })).user; + if (!user || user.deleted || user.is_bot) { + continue; + } + const opened = await context.sutReadClient.conversations.open({ + return_im: true, + users: `${driverUserId},${userId}`, + }); + const channelId = opened.channel?.id?.trim(); + if (!channelId) { + continue; + } + // Track ownership before the metadata call so outer cleanup can still + // close the MPIM when Slack rejects or times out during inspection. + openedChannelId = channelId; + const info = await context.sutReadClient.conversations.info({ channel: channelId }); + if (info.channel?.is_mpim && channelId.startsWith("C")) { + return { details: "opened C-prefixed MPIM", inputChannelId: channelId }; + } + await closeOpenedChannel(context); + } + throw new Error("Slack QA channel has no human member yielding a C-prefixed MPIM"); + }, + verifyObserved: ({ messages }) => { + const uniqueReplies = new Map(messages.map((message) => [message.ts, message])); + const matchingReplies = [...uniqueReplies.values()].filter((message) => + message.text.includes(token), + ); + if (uniqueReplies.size !== 1 || matchingReplies.length !== 1) { + throw new Error( + `expected one MPIM response with the marker, got ${uniqueReplies.size} response(s) and ${matchingReplies.length} marker match(es)`, + ); + } + return "one MPIM reply observed after message/app_mention twin delivery"; + }, + cleanup: closeOpenedChannel, + }; + }, +}; + +export const slackQaAllowlistBlockScenario: SlackQaScenarioImplementation = { + configOverrides: { + allowFrom: ["U_OPENCLAW_QA_NEVER_ALLOWED"], + users: ["U_OPENCLAW_QA_NEVER_ALLOWED"], + }, + buildRun: (sutUserId) => { + const token = `SLACK_QA_BLOCK_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + expectReply: false, + input: `<@${sutUserId}> reply with only this exact marker: ${token}`, + matchText: token, + }; + }, +}; + +export const slackQaChannelDisabledWarningScenario: SlackQaScenarioImplementation = { + configOverrides: { channelEnabled: false }, + buildRun: (sutUserId) => { + const marker = `SLACK_QA_DISABLED_${randomUUID().slice(0, 8).toUpperCase()}`; + let logCursor = 0; + return { + expectReply: false, + input: `<@${sutUserId}> reply with only this exact marker: ${marker}`, + matchText: marker, + preserveGatewayDebug: true, + beforeRun: async ({ gateway }) => { + const gatewayLogTail = (await gateway.call( + "logs.tail", + { limit: 1, maxBytes: 32_000 }, + { timeoutMs: SLACK_QA_LOG_TAIL_TIMEOUT_MS }, + )) as { cursor?: unknown }; + logCursor = typeof gatewayLogTail.cursor === "number" ? gatewayLogTail.cursor : 0; + }, + afterNoReply: async ({ gateway }) => { + const gatewayLogTail = (await gateway.call( + "logs.tail", + { cursor: logCursor, limit: 200, maxBytes: 256_000 }, + { timeoutMs: SLACK_QA_LOG_TAIL_TIMEOUT_MS }, + )) as { lines?: unknown }; + const gatewayLogLines = Array.isArray(gatewayLogTail.lines) + ? gatewayLogTail.lines.filter((line): line is string => typeof line === "string") + : []; + const expectedFields = [ + "Slack channel denied by configuration", + "channel_not_allowed", + "channel_disabled", + ]; + if ( + !gatewayLogLines.some((line) => expectedFields.every((field) => line.includes(field))) + ) { + throw new Error("disabled Slack channel did not emit the structured warning"); + } + return "structured disabled-channel warning observed"; + }, + }; + }, +}; + +export const slackQaTopLevelReplyShapeScenario: SlackQaScenarioImplementation = { + configOverrides: { replyToMode: "off" }, + buildRun: (sutUserId) => { + const token = `SLACK_QA_TOPLEVEL_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + expectReply: true, + input: `<@${sutUserId}> reply with only this exact marker: ${token}`, + matchText: token, + verify: (message) => { + if (message.thread_ts) { + throw new Error( + `expected top-level Slack reply without thread_ts; got ${message.thread_ts}`, + ); + } + }, + }; + }, +}; + +export const slackQaProgressCommentaryTrueScenario: SlackQaScenarioImplementation = { + configOverrides: { + progress: { commentary: true, toolProgress: false }, + }, + buildRun: (sutUserId) => + buildSlackProgressCommentaryRun(sutUserId, { + commentary: "draft", + toolProgress: "absent", + }), +}; + +export const slackQaProgressCommentaryFalseScenario: SlackQaScenarioImplementation = { + configOverrides: { + progress: { commentary: false, toolProgress: false }, + }, + buildRun: (sutUserId) => + buildSlackProgressCommentaryRun(sutUserId, { + commentary: "absent", + toolProgress: "absent", + }), +}; + +export const slackQaProgressCommentaryOmittedScenario: SlackQaScenarioImplementation = { + configOverrides: { + progress: { toolProgress: true }, + }, + buildRun: (sutUserId) => + buildSlackProgressCommentaryRun(sutUserId, { + commentary: "draft", + toolProgress: "draft", + }), +}; + +export const slackQaProgressCommentaryVerboseDedupeScenario: SlackQaScenarioImplementation = { + configOverrides: { + progress: { commentary: true, toolProgress: false, verboseDefault: "on" }, + }, + buildRun: (sutUserId) => + buildSlackProgressCommentaryRun(sutUserId, { + commentary: "standalone", + toolProgress: "standalone", + }), +}; + +export const slackQaChartPresentationNativeScenario: SlackQaScenarioImplementation = { + configOverrides: { messageTool: true }, + buildRun: (sutUserId) => { + const suffix = randomUUID().slice(0, 8).toUpperCase(); + const summaryText = `SLACK_QA_CHART_SUMMARY_${suffix}`; + const finalMarker = `SLACK_QA_CHART_DONE_${suffix}`; + const messageToolArgs = buildSlackChartMessageToolArgs(summaryText); + return { + expectReply: true, + input: [ + `<@${sutUserId}> Slack native chart QA check ${summaryText}.`, + `Call the message tool exactly once with these exact arguments: ${JSON.stringify(messageToolArgs)}.`, + `After the chart send succeeds, reply with only this exact marker: ${finalMarker}`, + ].join(" "), + matchText: finalMarker, + afterReply: async (_message, context) => { + await waitForSlackStoredMessage({ + channelId: context.channelId, + client: context.sutReadClient, + description: "message with native chart", + matchesMessage: (message) => + isExpectedSlackNativeChartMessage(message, renderSlackChartAccessibleText(summaryText)), + oldestTs: context.sentTs, + sutIdentity: context.sutIdentity, + timeoutMs: SLACK_QA_NATIVE_DATA_VERIFY_TIMEOUT_MS, + }); + return "verified native data_visualization block and deterministic accessible text"; + }, + }; + }, +}; + +export const slackQaTablePresentationNativeScenario: SlackQaScenarioImplementation = { + configOverrides: { messageTool: true }, + buildRun: (sutUserId) => { + const suffix = randomUUID().slice(0, 8).toUpperCase(); + const summaryText = `SLACK_QA_TABLE_SUMMARY_${suffix}`; + const finalMarker = `SLACK_QA_TABLE_DONE_${suffix}`; + const messageToolArgs = buildSlackTableMessageToolArgs(summaryText); + return { + expectReply: true, + input: [ + `<@${sutUserId}> Slack native table QA check ${summaryText}.`, + `Call the message tool exactly once with these exact arguments: ${JSON.stringify(messageToolArgs)}.`, + `After the table send succeeds, reply with only this exact marker: ${finalMarker}`, + ].join(" "), + matchText: finalMarker, + afterReply: async (_message, context) => { + await waitForSlackStoredMessage({ + channelId: context.channelId, + client: context.sutReadClient, + description: "message with native table", + matchesMessage: (message) => + isExpectedSlackNativeTableMessage(message, renderSlackTableAccessibleText(summaryText)), + oldestTs: context.sentTs, + sutIdentity: context.sutIdentity, + timeoutMs: SLACK_QA_NATIVE_DATA_VERIFY_TIMEOUT_MS, + }); + return "verified native data_table block and deterministic accessible text"; + }, + }; + }, +}; + +export const slackQaTableInvalidBlocksFallbackScenario: SlackQaScenarioImplementation = { + buildRun: () => ({ + kind: "direct-transport", + execute: runSlackTableInvalidBlocksFallbackScenario, + }), +}; + +export const slackQaReactionGlyphNativeScenario: SlackQaScenarioImplementation = { + configOverrides: { messageTool: true }, + buildRun: (sutUserId) => { + const token = `SLACK_QA_REACTION_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + expectReply: true, + input: [ + `<@${sutUserId}> use the message tool exactly once to react to this message.`, + 'Set action to "react", channel to "slack", and emoji to exactly "✅".', + "Do not substitute a shortcode.", + `After the reaction succeeds, reply with only this exact marker: ${token}`, + ].join(" "), + matchText: token, + afterReply: async (_message, context) => { + await waitForSlackReaction({ + channelId: context.channelId, + client: context.sutReadClient, + expectedReactionName: "white_check_mark", + messageId: context.sentTs, + sutUserId: context.sutIdentity.userId, + timeoutMs: SLACK_QA_REACTION_VERIFY_TIMEOUT_MS, + }); + return "verified SUT white_check_mark reaction from exact glyph instruction"; + }, + }; + }, +}; + +export const slackQaApprovalExecNativeScenario: SlackQaScenarioImplementation = { + configOverrides: { + approvals: { + exec: true, + target: "channel", + }, + }, + buildRun: () => ({ + approvalKind: "exec", + decision: "allow-once", + kind: "approval", + token: `SLACK_QA_EXEC_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`, + }), +}; + +export const slackQaApprovalPluginNativeScenario: SlackQaScenarioImplementation = { + configOverrides: { + approvals: { + exec: true, + plugin: true, + target: "channel", + }, + }, + buildRun: () => ({ + approvalKind: "plugin", + decision: "allow-once", + kind: "approval", + token: `SLACK_QA_PLUGIN_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`, + }), +}; + +export const slackQaCodexApprovalExecNativeScenario: SlackQaScenarioImplementation = { + configOverrides: { + approvals: { + exec: true, + plugin: true, + target: "channel", + }, + codexApproval: true, + }, + buildRun: () => ({ + approvalKind: "plugin", + appServerMethod: "item/commandExecution/requestApproval", + decision: "allow-once", + kind: "codex-approval", + token: `SLACK_QA_CODEX_EXEC_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`, + }), +}; + +export const slackQaCodexApprovalPluginNativeScenario: SlackQaScenarioImplementation = { + configOverrides: { + approvals: { + exec: true, + plugin: true, + target: "channel", + }, + codexApproval: true, + }, + buildRun: () => ({ + approvalKind: "plugin", + appServerMethod: "item/fileChange/requestApproval", + decision: "allow-once", + kind: "codex-approval", + token: `SLACK_QA_CODEX_FILE_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`, + }), +}; diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.scenarios.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.scenarios.ts deleted file mode 100644 index 9932f84f2bb6..000000000000 --- a/extensions/qa-lab/src/live-transports/slack/slack-live.scenarios.ts +++ /dev/null @@ -1,476 +0,0 @@ -// QA Lab Slack live scenario catalog. -import { randomUUID } from "node:crypto"; -import { setTimeout as sleep } from "node:timers/promises"; -import { waitForSlackReaction } from "./slack-live.codex-approval.js"; -import { - SLACK_QA_REACTION_VERIFY_TIMEOUT_MS, - SLACK_QA_NATIVE_DATA_VERIFY_TIMEOUT_MS, - SLACK_QA_LOG_TAIL_TIMEOUT_MS, - type SlackQaScenarioDefinition, - type SlackQaScenarioContext, -} from "./slack-live.contracts.js"; -import { - isExpectedSlackNativeChartMessage, - isExpectedSlackNativeTableMessage, - runSlackTableInvalidBlocksFallbackScenario, - waitForSlackStoredMessage, -} from "./slack-live.observations.js"; -import { - buildSlackChartMessageToolArgs, - renderSlackChartAccessibleText, - buildSlackTableMessageToolArgs, - renderSlackTableAccessibleText, - buildSlackProgressCommentaryRun, -} from "./slack-live.scenario-fixtures.js"; - -const SLACK_QA_SCENARIOS: SlackQaScenarioDefinition[] = [ - { - id: "slack-canary", - title: "Slack canary echo", - timeoutMs: 45_000, - buildRun: (sutUserId) => { - const token = `SLACK_QA_ECHO_${randomUUID().slice(0, 8).toUpperCase()}`; - return { - expectReply: true, - input: `<@${sutUserId}> reply with only this exact marker: ${token}`, - matchText: token, - }; - }, - }, - { - id: "slack-mention-gating", - title: "Slack unmentioned bot message does not trigger", - timeoutMs: 8_000, - buildRun: () => { - const token = `SLACK_QA_NOMENTION_${randomUUID().slice(0, 8).toUpperCase()}`; - return { - expectReply: false, - input: `reply with only this exact marker: ${token}`, - matchText: token, - }; - }, - }, - { - id: "slack-mpim-app-mention-dedupe", - title: "Slack MPIM app mention dispatches once", - timeoutMs: 90_000, - configOverrides: { groupDmEnabled: true }, - buildRun: (sutUserId) => { - const token = `SLACK_QA_MPIM_${randomUUID().slice(0, 8).toUpperCase()}`; - let openedChannelId: string | undefined; - const closeOpenedChannel = async (context: Omit) => { - if (!openedChannelId) { - return; - } - const channelId = openedChannelId; - for (let attempt = 1; attempt <= 2; attempt += 1) { - try { - await context.sutReadClient.conversations.close({ channel: channelId }); - openedChannelId = undefined; - return; - } catch (error) { - if (attempt === 2) { - throw error; - } - // Retain ownership until Slack confirms closure; one bounded retry - // covers a transient API failure without hiding a leaked MPIM. - await sleep(500); - } - } - }; - return { - expectReply: true, - input: `<@${sutUserId}> reply with only this exact marker: ${token}`, - matchText: token, - settleObservedMs: 60_000, - beforeRun: async (context) => { - const driverAuth = await context.driverClient.auth.test(); - const driverUserId = driverAuth.user_id?.trim(); - if (!driverUserId) { - throw new Error("Slack QA driver auth.test returned no user_id"); - } - const members = await context.sutReadClient.conversations.members({ - channel: context.channelId, - limit: 100, - }); - const candidateUserIds = (members.members ?? []).filter( - (userId) => userId !== driverUserId && userId !== context.sutIdentity.userId, - ); - for (const userId of candidateUserIds) { - const user = (await context.sutReadClient.users.info({ user: userId })).user; - if (!user || user.deleted || user.is_bot) { - continue; - } - const opened = await context.sutReadClient.conversations.open({ - return_im: true, - users: `${driverUserId},${userId}`, - }); - const channelId = opened.channel?.id?.trim(); - if (!channelId) { - continue; - } - // Track ownership before the metadata call so outer cleanup can still - // close the MPIM when Slack rejects or times out during inspection. - openedChannelId = channelId; - const info = await context.sutReadClient.conversations.info({ channel: channelId }); - if (info.channel?.is_mpim && channelId.startsWith("C")) { - return { details: "opened C-prefixed MPIM", inputChannelId: channelId }; - } - await closeOpenedChannel(context); - } - throw new Error("Slack QA channel has no human member yielding a C-prefixed MPIM"); - }, - verifyObserved: ({ messages }) => { - const uniqueReplies = new Map(messages.map((message) => [message.ts, message])); - const matchingReplies = [...uniqueReplies.values()].filter((message) => - message.text.includes(token), - ); - if (uniqueReplies.size !== 1 || matchingReplies.length !== 1) { - throw new Error( - `expected one MPIM response with the marker, got ${uniqueReplies.size} response(s) and ${matchingReplies.length} marker match(es)`, - ); - } - return "one MPIM reply observed after message/app_mention twin delivery"; - }, - cleanup: closeOpenedChannel, - }; - }, - }, - { - id: "slack-allowlist-block", - title: "Slack non-allowlisted sender does not trigger", - timeoutMs: 8_000, - configOverrides: { - allowFrom: ["U_OPENCLAW_QA_NEVER_ALLOWED"], - users: ["U_OPENCLAW_QA_NEVER_ALLOWED"], - }, - buildRun: (sutUserId) => { - const token = `SLACK_QA_BLOCK_${randomUUID().slice(0, 8).toUpperCase()}`; - return { - expectReply: false, - input: `<@${sutUserId}> reply with only this exact marker: ${token}`, - matchText: token, - }; - }, - }, - { - id: "slack-channel-disabled-warning", - title: "Slack disabled channel warns and does not trigger", - timeoutMs: 8_000, - configOverrides: { channelEnabled: false }, - buildRun: (sutUserId) => { - const marker = `SLACK_QA_DISABLED_${randomUUID().slice(0, 8).toUpperCase()}`; - let logCursor = 0; - return { - expectReply: false, - input: `<@${sutUserId}> reply with only this exact marker: ${marker}`, - matchText: marker, - preserveGatewayDebug: true, - beforeRun: async ({ gateway }) => { - const gatewayLogTail = (await gateway.call( - "logs.tail", - { limit: 1, maxBytes: 32_000 }, - { timeoutMs: SLACK_QA_LOG_TAIL_TIMEOUT_MS }, - )) as { cursor?: unknown }; - logCursor = typeof gatewayLogTail.cursor === "number" ? gatewayLogTail.cursor : 0; - }, - afterNoReply: async ({ gateway }) => { - const gatewayLogTail = (await gateway.call( - "logs.tail", - { cursor: logCursor, limit: 200, maxBytes: 256_000 }, - { timeoutMs: SLACK_QA_LOG_TAIL_TIMEOUT_MS }, - )) as { lines?: unknown }; - const gatewayLogLines = Array.isArray(gatewayLogTail.lines) - ? gatewayLogTail.lines.filter((line): line is string => typeof line === "string") - : []; - const expectedFields = [ - "Slack channel denied by configuration", - "channel_not_allowed", - "channel_disabled", - ]; - if ( - !gatewayLogLines.some((line) => expectedFields.every((field) => line.includes(field))) - ) { - throw new Error("disabled Slack channel did not emit the structured warning"); - } - return "structured disabled-channel warning observed"; - }, - }; - }, - }, - { - id: "slack-top-level-reply-shape", - title: "Slack top-level reply stays top-level", - timeoutMs: 45_000, - configOverrides: { replyToMode: "off" }, - buildRun: (sutUserId) => { - const token = `SLACK_QA_TOPLEVEL_${randomUUID().slice(0, 8).toUpperCase()}`; - return { - expectReply: true, - input: `<@${sutUserId}> reply with only this exact marker: ${token}`, - matchText: token, - verify: (message) => { - if (message.thread_ts) { - throw new Error( - `expected top-level Slack reply without thread_ts; got ${message.thread_ts}`, - ); - } - }, - }; - }, - }, - { - id: "slack-progress-commentary-true", - title: "Slack progress commentary true is independent from tool progress", - timeoutMs: 90_000, - configOverrides: { - progress: { commentary: true, toolProgress: false }, - }, - buildRun: (sutUserId) => - buildSlackProgressCommentaryRun(sutUserId, { - commentary: "draft", - toolProgress: "absent", - }), - }, - { - id: "slack-progress-commentary-false", - title: "Slack progress commentary false stays out of the progress draft", - timeoutMs: 90_000, - configOverrides: { - progress: { commentary: false, toolProgress: false }, - }, - buildRun: (sutUserId) => - buildSlackProgressCommentaryRun(sutUserId, { - commentary: "absent", - toolProgress: "absent", - }), - }, - { - id: "slack-progress-commentary-omitted", - title: "Slack omitted progress commentary preserves the tool-progress default", - timeoutMs: 90_000, - configOverrides: { - progress: { toolProgress: true }, - }, - buildRun: (sutUserId) => - buildSlackProgressCommentaryRun(sutUserId, { - commentary: "draft", - toolProgress: "draft", - }), - }, - { - id: "slack-progress-commentary-verbose-dedupe", - title: "Slack explicit commentary yields to durable verbose progress", - timeoutMs: 90_000, - configOverrides: { - progress: { commentary: true, toolProgress: false, verboseDefault: "on" }, - }, - buildRun: (sutUserId) => - buildSlackProgressCommentaryRun(sutUserId, { - commentary: "standalone", - toolProgress: "standalone", - }), - }, - { - id: "slack-chart-presentation-native", - title: "Slack portable chart renders as a native data visualization", - timeoutMs: 90_000, - configOverrides: { messageTool: true }, - buildRun: (sutUserId) => { - const suffix = randomUUID().slice(0, 8).toUpperCase(); - const summaryText = `SLACK_QA_CHART_SUMMARY_${suffix}`; - const finalMarker = `SLACK_QA_CHART_DONE_${suffix}`; - const messageToolArgs = buildSlackChartMessageToolArgs(summaryText); - return { - expectReply: true, - input: [ - `<@${sutUserId}> Slack native chart QA check ${summaryText}.`, - `Call the message tool exactly once with these exact arguments: ${JSON.stringify(messageToolArgs)}.`, - `After the chart send succeeds, reply with only this exact marker: ${finalMarker}`, - ].join(" "), - matchText: finalMarker, - afterReply: async (_message, context) => { - await waitForSlackStoredMessage({ - channelId: context.channelId, - client: context.sutReadClient, - description: "message with native chart", - matchesMessage: (message) => - isExpectedSlackNativeChartMessage( - message, - renderSlackChartAccessibleText(summaryText), - ), - oldestTs: context.sentTs, - sutIdentity: context.sutIdentity, - timeoutMs: SLACK_QA_NATIVE_DATA_VERIFY_TIMEOUT_MS, - }); - return "verified native data_visualization block and deterministic accessible text"; - }, - }; - }, - }, - { - id: "slack-table-presentation-native", - title: "Slack portable table renders as a native data table", - timeoutMs: 90_000, - configOverrides: { messageTool: true }, - buildRun: (sutUserId) => { - const suffix = randomUUID().slice(0, 8).toUpperCase(); - const summaryText = `SLACK_QA_TABLE_SUMMARY_${suffix}`; - const finalMarker = `SLACK_QA_TABLE_DONE_${suffix}`; - const messageToolArgs = buildSlackTableMessageToolArgs(summaryText); - return { - expectReply: true, - input: [ - `<@${sutUserId}> Slack native table QA check ${summaryText}.`, - `Call the message tool exactly once with these exact arguments: ${JSON.stringify(messageToolArgs)}.`, - `After the table send succeeds, reply with only this exact marker: ${finalMarker}`, - ].join(" "), - matchText: finalMarker, - afterReply: async (_message, context) => { - await waitForSlackStoredMessage({ - channelId: context.channelId, - client: context.sutReadClient, - description: "message with native table", - matchesMessage: (message) => - isExpectedSlackNativeTableMessage( - message, - renderSlackTableAccessibleText(summaryText), - ), - oldestTs: context.sentTs, - sutIdentity: context.sutIdentity, - timeoutMs: SLACK_QA_NATIVE_DATA_VERIFY_TIMEOUT_MS, - }); - return "verified native data_table block and deterministic accessible text"; - }, - }; - }, - }, - { - id: "slack-table-invalid-blocks-fallback", - title: "Slack rejects an over-limit native table and stores its complete fallback", - timeoutMs: 45_000, - buildRun: () => ({ - kind: "direct-transport", - execute: runSlackTableInvalidBlocksFallbackScenario, - }), - }, - { - id: "slack-reaction-glyph-native", - title: "Slack message tool normalizes an emoji glyph reaction", - timeoutMs: 90_000, - configOverrides: { messageTool: true }, - buildRun: (sutUserId) => { - const token = `SLACK_QA_REACTION_${randomUUID().slice(0, 8).toUpperCase()}`; - return { - expectReply: true, - input: [ - `<@${sutUserId}> use the message tool exactly once to react to this message.`, - 'Set action to "react", channel to "slack", and emoji to exactly "✅".', - "Do not substitute a shortcode.", - `After the reaction succeeds, reply with only this exact marker: ${token}`, - ].join(" "), - matchText: token, - afterReply: async (_message, context) => { - await waitForSlackReaction({ - channelId: context.channelId, - client: context.sutReadClient, - expectedReactionName: "white_check_mark", - messageId: context.sentTs, - sutUserId: context.sutIdentity.userId, - timeoutMs: SLACK_QA_REACTION_VERIFY_TIMEOUT_MS, - }); - return "verified SUT white_check_mark reaction from exact glyph instruction"; - }, - }; - }, - }, - { - id: "slack-approval-exec-native", - title: "Slack native exec approval prompt resolves", - timeoutMs: 60_000, - configOverrides: { - approvals: { - exec: true, - target: "channel", - }, - }, - buildRun: () => ({ - approvalKind: "exec", - decision: "allow-once", - kind: "approval", - token: `SLACK_QA_EXEC_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`, - }), - }, - { - id: "slack-approval-plugin-native", - title: "Slack native plugin approval prompt resolves with exec approvals enabled", - timeoutMs: 60_000, - configOverrides: { - approvals: { - exec: true, - plugin: true, - target: "channel", - }, - }, - buildRun: () => ({ - approvalKind: "plugin", - decision: "allow-once", - kind: "approval", - token: `SLACK_QA_PLUGIN_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`, - }), - }, - { - id: "slack-codex-approval-exec-native", - title: "Slack native Codex command approval prompt resolves", - timeoutMs: 180_000, - configOverrides: { - approvals: { - exec: true, - plugin: true, - target: "channel", - }, - codexApproval: true, - }, - forcedRuntime: "codex", - buildRun: () => ({ - approvalKind: "plugin", - appServerMethod: "item/commandExecution/requestApproval", - decision: "allow-once", - kind: "codex-approval", - token: `SLACK_QA_CODEX_EXEC_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`, - }), - }, - { - id: "slack-codex-approval-plugin-native", - title: "Slack native Codex file approval prompt resolves", - timeoutMs: 180_000, - configOverrides: { - approvals: { - exec: true, - plugin: true, - target: "channel", - }, - codexApproval: true, - }, - forcedRuntime: "codex", - buildRun: () => ({ - approvalKind: "plugin", - appServerMethod: "item/fileChange/requestApproval", - decision: "allow-once", - kind: "codex-approval", - token: `SLACK_QA_CODEX_FILE_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`, - }), - }, -]; - -export function listSlackQaScenarioCatalog() { - return SLACK_QA_SCENARIOS.map((scenario) => ({ id: scenario.id })); -} - -export function getSlackQaScenarioDefinition(id: string) { - const scenario = SLACK_QA_SCENARIOS.find((candidate) => candidate.id === id); - if (!scenario) { - throw new Error(`unknown Slack QA scenario id: ${id}`); - } - return scenario; -} diff --git a/extensions/qa-lab/src/live-transports/whatsapp/scenario-environment.ts b/extensions/qa-lab/src/live-transports/whatsapp/scenario-environment.ts index 411bdb30dfc2..0ba9e3d73842 100644 --- a/extensions/qa-lab/src/live-transports/whatsapp/scenario-environment.ts +++ b/extensions/qa-lab/src/live-transports/whatsapp/scenario-environment.ts @@ -11,8 +11,9 @@ import { resolveWhatsAppQaScenarioTarget, type WhatsAppObservedMessage, type WhatsAppQaRuntimeEnv, + type WhatsAppQaScenarioImplementation, + type WhatsAppQaScenarioRun, } from "./whatsapp-live.contracts.js"; -import { getWhatsAppQaScenarioDefinition } from "./whatsapp-live.scenarios.js"; import { waitForWhatsAppChannelStable } from "./whatsapp-live.setup.js"; type AdapterFactory = NonNullable; @@ -20,12 +21,16 @@ type AdapterDefinition = Awaited>; type FlowPreparationInput = Parameters>[0]; export type WhatsAppQaScenarioEnvironment = { + configureScenario: (implementation: WhatsAppQaScenarioImplementation) => Promise<{ + run: WhatsAppQaScenarioRun; + }>; driverAuthDir: string; gateway: FlowPreparationInput["gateway"]; getDriver: () => WhatsAppQaDriverSession; observedMessages: WhatsAppObservedMessage[]; replaceDriver: (driver: WhatsAppQaDriverSession) => Promise; runtimeEnv: WhatsAppQaRuntimeEnv; + scenario: { id: string; timeoutMs: number; title: string }; sutAccountId: string; sutAuthDir: string; }; @@ -55,70 +60,73 @@ export function createWhatsAppQaScenarioEnvironment(params: { const observedMessages: WhatsAppObservedMessage[] = []; const prepareFlow = async (input: FlowPreparationInput) => { - const scenarioId = input.config.whatsappScenarioId; - if (typeof scenarioId !== "string") { - return undefined; - } - const scenario = getWhatsAppQaScenarioDefinition(scenarioId); - if (scenario.requiresGroupJid && !params.runtimeEnv.groupJid) { - if (params.explicitScenarioSelection) { - throw new Error( - `Requested WhatsApp scenario ${scenario.id} requires groupJid in the credential payload`, - ); - } - throw new QaSuiteScenarioSkipError( - `WhatsApp scenario ${scenario.id} requires groupJid in the credential payload`, - ); - } - const scenarioRun = scenario.buildRun(); - const resolvedTarget = resolveWhatsAppQaScenarioTarget({ - groupJid: params.runtimeEnv.groupJid, - scenarioId: scenario.id, - target: scenarioRun.kind === "approval" ? (scenarioRun.target ?? "dm") : scenarioRun.target, - }); - const groupJid = resolvedTarget.target === "group" ? resolvedTarget.groupJid : undefined; - const allowFrom = - scenarioRun.kind === "approval" - ? [params.runtimeEnv.driverPhoneE164] - : scenarioRun.configMode === "open" - ? ["*"] - : scenarioRun.configMode === "pairing" - ? ["+15550000000"] - : [params.runtimeEnv.driverPhoneE164]; - const dmPolicy = - scenarioRun.kind === "approval" - ? "allowlist" - : scenarioRun.configMode === "open" || scenarioRun.configMode === "disabled" - ? scenarioRun.configMode - : scenarioRun.configMode === "allowlist" - ? "allowlist" - : "pairing"; - const snapshot = await readLiveQaGatewayConfig(input.gateway); - const cfg = buildWhatsAppQaConfig(snapshot.config as OpenClawConfig, { - allowFrom, - authDir: params.sutAuthDir, - dmPolicy, - groupJid, - ownerAllowFrom: [params.runtimeEnv.driverPhoneE164], - overrides: scenario.configOverrides, - sutAccountId: params.accountId, - }); - await patchLiveQaGatewayConfig({ - gateway: input.gateway, - patch: cfg as Record, - replacePaths: resolveWhatsAppQaReplacePaths(params.accountId), - timeoutMs: input.timeoutMs, - waitForConfigRestartSettle: input.waitForConfigRestartSettle, - }); - await waitForWhatsAppChannelStable(input.gateway as never, params.accountId); return { whatsappScenarioContext: { + configureScenario: async (implementation: WhatsAppQaScenarioImplementation) => { + if (implementation.requiresGroupJid && !params.runtimeEnv.groupJid) { + if (params.explicitScenarioSelection) { + throw new Error( + `Requested WhatsApp scenario ${input.scenarioId} requires groupJid in the credential payload`, + ); + } + throw new QaSuiteScenarioSkipError( + `WhatsApp scenario ${input.scenarioId} requires groupJid in the credential payload`, + ); + } + const run = implementation.buildRun(); + const resolvedTarget = resolveWhatsAppQaScenarioTarget({ + groupJid: params.runtimeEnv.groupJid, + scenarioId: input.scenarioId, + target: run.kind === "approval" ? (run.target ?? "dm") : run.target, + }); + const groupJid = resolvedTarget.target === "group" ? resolvedTarget.groupJid : undefined; + const allowFrom = + run.kind === "approval" + ? [params.runtimeEnv.driverPhoneE164] + : run.configMode === "open" + ? ["*"] + : run.configMode === "pairing" + ? ["+15550000000"] + : [params.runtimeEnv.driverPhoneE164]; + const dmPolicy = + run.kind === "approval" + ? "allowlist" + : run.configMode === "open" || run.configMode === "disabled" + ? run.configMode + : run.configMode === "allowlist" + ? "allowlist" + : "pairing"; + const snapshot = await readLiveQaGatewayConfig(input.gateway); + const cfg = buildWhatsAppQaConfig(snapshot.config as OpenClawConfig, { + allowFrom, + authDir: params.sutAuthDir, + dmPolicy, + groupJid, + ownerAllowFrom: [params.runtimeEnv.driverPhoneE164], + overrides: implementation.configOverrides, + sutAccountId: params.accountId, + }); + await patchLiveQaGatewayConfig({ + gateway: input.gateway, + patch: cfg as Record, + replacePaths: resolveWhatsAppQaReplacePaths(params.accountId), + timeoutMs: input.timeoutMs, + waitForConfigRestartSettle: input.waitForConfigRestartSettle, + }); + await waitForWhatsAppChannelStable(input.gateway as never, params.accountId); + return { run }; + }, driverAuthDir: params.driverAuthDir, gateway: input.gateway, getDriver: params.getDriver, observedMessages, replaceDriver: params.replaceDriver, runtimeEnv: params.runtimeEnv, + scenario: { + id: input.scenarioId, + timeoutMs: input.timeoutMs, + title: input.scenarioTitle, + }, sutAccountId: params.accountId, sutAuthDir: params.sutAuthDir, } satisfies WhatsAppQaScenarioEnvironment, diff --git a/extensions/qa-lab/src/live-transports/whatsapp/scenario-runtime.ts b/extensions/qa-lab/src/live-transports/whatsapp/scenario-runtime.ts index 6e04d3af06a2..7ce52ef66884 100644 --- a/extensions/qa-lab/src/live-transports/whatsapp/scenario-runtime.ts +++ b/extensions/qa-lab/src/live-transports/whatsapp/scenario-runtime.ts @@ -7,8 +7,10 @@ import { resolveWhatsAppQaScenarioTarget, type WhatsAppObservedMessage, type WhatsAppQaMessageScenarioContext, - type WhatsAppQaScenarioDefinition, + type WhatsAppQaScenarioImplementation, + type WhatsAppQaScenarioMetadata, type WhatsAppQaScenarioResult, + type WhatsAppQaScenarioRun, } from "./whatsapp-live.contracts.js"; import { WHATSAPP_QA_TRANSIENT_DRIVER_ATTEMPTS, @@ -20,16 +22,64 @@ import { waitForNoWhatsAppReply, waitForWhatsAppScenarioSutMessage, } from "./whatsapp-live.operations.js"; -import { getWhatsAppQaScenarioDefinition } from "./whatsapp-live.scenarios.js"; import { waitForWhatsAppChannelStable } from "./whatsapp-live.setup.js"; +export { + whatsappQaGroupAudioGatingScenario, + whatsappQaGroupOutboundAudioScenario, + whatsappQaGroupOutboundMediaScenario, + whatsappQaGroupOutboundPollScenario, + whatsappQaInboundStructuredMessagesScenario, + whatsappQaMessageActionsScenario, + whatsappQaOutboundDocumentPreservesFilenameScenario, + whatsappQaOutboundPollScenario, + whatsappQaOutboundSendSerializationScenario, +} from "./whatsapp-live.scenario-implementations.capabilities.js"; +export { + whatsappQaBroadcastGroupFanoutScenario, + whatsappQaCanaryScenario, + whatsappQaGroupActivationAlwaysScenario, + whatsappQaGroupPendingHistoryContextScenario, + whatsappQaGroupReplyToBotTriggersScenario, + whatsappQaGroupReplyToMessageScenario, + whatsappQaMentionGatingScenario, + whatsappQaReplyToMessageScenario, + whatsappQaReplyToModeBatchedScenario, + whatsappQaTopLevelReplyShapeScenario, +} from "./whatsapp-live.scenario-implementations.conversation.js"; +export { + whatsappQaApprovalExecDenyNativeScenario, + whatsappQaApprovalExecGroupReactionNativeScenario, + whatsappQaApprovalExecNativeScenario, + whatsappQaApprovalExecReactionNativeScenario, + whatsappQaApprovalPluginNativeScenario, + whatsappQaGroupAllowlistBlockScenario, + whatsappQaReplyDeliveryShapeScenario, + whatsappQaStatusReactionLifecycleScenario, + whatsappQaStatusReactionsScenario, + whatsappQaStreamFinalMessageAccountingScenario, +} from "./whatsapp-live.scenario-implementations.delivery.js"; +export { + whatsappQaAgentMessageActionReactScenario, + whatsappQaAgentMessageActionUploadFileScenario, + whatsappQaAudioPreflightScenario, + whatsappQaGroupAgentMessageActionReactScenario, + whatsappQaGroupAgentMessageActionUploadFileScenario, + whatsappQaInboundImageCaptionScenario, + whatsappQaInboundReactionNoTriggerScenario, + whatsappQaOutboundMediaMatrixScenario, + whatsappQaReplyContextIsolationScenario, +} from "./whatsapp-live.scenario-implementations.user-path.js"; + async function runWhatsAppScenarioAttempt(params: { environment: WhatsAppQaScenarioEnvironment; - scenario: WhatsAppQaScenarioDefinition; + implementation: WhatsAppQaScenarioImplementation; + run: WhatsAppQaScenarioRun; + scenario: WhatsAppQaScenarioMetadata; }): Promise { const driver = params.environment.getDriver(); const runtimeEnv = params.environment.runtimeEnv; - const scenarioRun = params.scenario.buildRun(); + const scenarioRun = params.run; const resolvedTarget = resolveWhatsAppQaScenarioTarget({ groupJid: runtimeEnv.groupJid, scenarioId: params.scenario.id, @@ -61,7 +111,7 @@ async function runWhatsAppScenarioAttempt(params: { turnSourceTo: approvalTurnSourceTo, }); return { - ...buildWhatsAppQaScenarioResultBase(params.scenario), + ...buildWhatsAppQaScenarioResultBase(params.scenario, params.implementation), status: "pass", details: `${scenarioRun.approvalKind} approval ${approval.approvalId} resolved ${scenarioRun.decision} in ${approval.rttMs}ms`, rttMs: approval.rttMs, @@ -164,7 +214,7 @@ async function runWhatsAppScenarioAttempt(params: { }), }); return { - ...buildWhatsAppQaScenarioResultBase(params.scenario), + ...buildWhatsAppQaScenarioResultBase(params.scenario, params.implementation), status: "pass", details: ["no reply", afterSendDetails].filter(Boolean).join("; "), }; @@ -186,7 +236,7 @@ async function runWhatsAppScenarioAttempt(params: { const responseObservedAt = new Date(reply.observedAt); const rttMs = responseObservedAt.getTime() - requestStartedAt.getTime(); return { - ...buildWhatsAppQaScenarioResultBase(params.scenario), + ...buildWhatsAppQaScenarioResultBase(params.scenario, params.implementation), status: "pass", details: [`reply matched in ${rttMs}ms`, afterSendDetails, afterReplyDetails, batchDetails] .filter(Boolean) @@ -203,11 +253,23 @@ async function runWhatsAppScenarioAttempt(params: { }; } -async function runWhatsAppScenario(environment: WhatsAppQaScenarioEnvironment, scenarioId: string) { - const scenario = getWhatsAppQaScenarioDefinition(scenarioId); +export async function runWhatsAppScenario( + environment: WhatsAppQaScenarioEnvironment, + implementation: WhatsAppQaScenarioImplementation, +) { + const scenario = environment.scenario; + const { run: configuredRun } = await environment.configureScenario(implementation); for (let attempt = 1; attempt <= WHATSAPP_QA_TRANSIENT_DRIVER_ATTEMPTS; attempt += 1) { try { - const result = await runWhatsAppScenarioAttempt({ environment, scenario }); + // Retry with fresh markers and callback state while retaining the gateway config + // prepared from the equivalent first run. + const run = attempt === 1 ? configuredRun : implementation.buildRun(); + const result = await runWhatsAppScenarioAttempt({ + environment, + implementation, + run, + scenario, + }); return attempt === 1 ? result : { ...result, details: `${result.details}; driver reconnected ${attempt - 1}x` }; @@ -225,96 +287,5 @@ async function runWhatsAppScenario(environment: WhatsAppQaScenarioEnvironment, s await environment.replaceDriver(nextDriver); } } - throw new Error(`WhatsApp scenario ${scenarioId} exhausted driver retries`); + throw new Error(`WhatsApp scenario ${scenario.id} exhausted driver retries`); } - -export const runWhatsAppCanaryScenario = (context: WhatsAppQaScenarioEnvironment) => - runWhatsAppScenario(context, "whatsapp-canary"); -export const runWhatsAppMentionGatingScenario = (context: WhatsAppQaScenarioEnvironment) => - runWhatsAppScenario(context, "whatsapp-mention-gating"); -export const runWhatsAppGroupPendingHistoryContextScenario = ( - context: WhatsAppQaScenarioEnvironment, -) => runWhatsAppScenario(context, "whatsapp-group-pending-history-context"); -export const runWhatsAppBroadcastGroupFanoutScenario = (context: WhatsAppQaScenarioEnvironment) => - runWhatsAppScenario(context, "whatsapp-broadcast-group-fanout"); -export const runWhatsAppGroupActivationAlwaysScenario = (context: WhatsAppQaScenarioEnvironment) => - runWhatsAppScenario(context, "whatsapp-group-activation-always"); -export const runWhatsAppGroupReplyToBotTriggersScenario = ( - context: WhatsAppQaScenarioEnvironment, -) => runWhatsAppScenario(context, "whatsapp-group-reply-to-bot-triggers"); -export const runWhatsAppTopLevelReplyShapeScenario = (context: WhatsAppQaScenarioEnvironment) => - runWhatsAppScenario(context, "whatsapp-top-level-reply-shape"); -export const runWhatsAppReplyToMessageScenario = (context: WhatsAppQaScenarioEnvironment) => - runWhatsAppScenario(context, "whatsapp-reply-to-message"); -export const runWhatsAppGroupReplyToMessageScenario = (context: WhatsAppQaScenarioEnvironment) => - runWhatsAppScenario(context, "whatsapp-group-reply-to-message"); -export const runWhatsAppReplyToModeBatchedScenario = (context: WhatsAppQaScenarioEnvironment) => - runWhatsAppScenario(context, "whatsapp-reply-to-mode-batched"); -export const runWhatsAppAgentMessageActionReactScenario = ( - context: WhatsAppQaScenarioEnvironment, -) => runWhatsAppScenario(context, "whatsapp-agent-message-action-react"); -export const runWhatsAppAgentMessageActionUploadFileScenario = ( - context: WhatsAppQaScenarioEnvironment, -) => runWhatsAppScenario(context, "whatsapp-agent-message-action-upload-file"); -export const runWhatsAppGroupAgentMessageActionReactScenario = ( - context: WhatsAppQaScenarioEnvironment, -) => runWhatsAppScenario(context, "whatsapp-group-agent-message-action-react"); -export const runWhatsAppGroupAgentMessageActionUploadFileScenario = ( - context: WhatsAppQaScenarioEnvironment, -) => runWhatsAppScenario(context, "whatsapp-group-agent-message-action-upload-file"); -export const runWhatsAppInboundReactionNoTriggerScenario = ( - context: WhatsAppQaScenarioEnvironment, -) => runWhatsAppScenario(context, "whatsapp-inbound-reaction-no-trigger"); -export const runWhatsAppReplyContextIsolationScenario = (context: WhatsAppQaScenarioEnvironment) => - runWhatsAppScenario(context, "whatsapp-reply-context-isolation"); -export const runWhatsAppInboundImageCaptionScenario = (context: WhatsAppQaScenarioEnvironment) => - runWhatsAppScenario(context, "whatsapp-inbound-image-caption"); -export const runWhatsAppAudioPreflightScenario = (context: WhatsAppQaScenarioEnvironment) => - runWhatsAppScenario(context, "whatsapp-audio-preflight"); -export const runWhatsAppOutboundMediaMatrixScenario = (context: WhatsAppQaScenarioEnvironment) => - runWhatsAppScenario(context, "whatsapp-outbound-media-matrix"); -export const runWhatsAppOutboundDocumentPreservesFilenameScenario = ( - context: WhatsAppQaScenarioEnvironment, -) => runWhatsAppScenario(context, "whatsapp-outbound-document-preserves-filename"); -export const runWhatsAppOutboundSendSerializationScenario = ( - context: WhatsAppQaScenarioEnvironment, -) => runWhatsAppScenario(context, "whatsapp-outbound-send-serialization"); -export const runWhatsAppOutboundPollScenario = (context: WhatsAppQaScenarioEnvironment) => - runWhatsAppScenario(context, "whatsapp-outbound-poll"); -export const runWhatsAppGroupOutboundMediaScenario = (context: WhatsAppQaScenarioEnvironment) => - runWhatsAppScenario(context, "whatsapp-group-outbound-media"); -export const runWhatsAppGroupOutboundAudioScenario = (context: WhatsAppQaScenarioEnvironment) => - runWhatsAppScenario(context, "whatsapp-group-outbound-audio"); -export const runWhatsAppGroupOutboundPollScenario = (context: WhatsAppQaScenarioEnvironment) => - runWhatsAppScenario(context, "whatsapp-group-outbound-poll"); -export const runWhatsAppMessageActionsScenario = (context: WhatsAppQaScenarioEnvironment) => - runWhatsAppScenario(context, "whatsapp-message-actions"); -export const runWhatsAppInboundStructuredMessagesScenario = ( - context: WhatsAppQaScenarioEnvironment, -) => runWhatsAppScenario(context, "whatsapp-inbound-structured-messages"); -export const runWhatsAppGroupAudioGatingScenario = (context: WhatsAppQaScenarioEnvironment) => - runWhatsAppScenario(context, "whatsapp-group-audio-gating"); -export const runWhatsAppReplyDeliveryShapeScenario = (context: WhatsAppQaScenarioEnvironment) => - runWhatsAppScenario(context, "whatsapp-reply-delivery-shape"); -export const runWhatsAppStreamFinalMessageAccountingScenario = ( - context: WhatsAppQaScenarioEnvironment, -) => runWhatsAppScenario(context, "whatsapp-stream-final-message-accounting"); -export const runWhatsAppApprovalExecDenyNativeScenario = (context: WhatsAppQaScenarioEnvironment) => - runWhatsAppScenario(context, "whatsapp-approval-exec-deny-native"); -export const runWhatsAppStatusReactionsScenario = (context: WhatsAppQaScenarioEnvironment) => - runWhatsAppScenario(context, "whatsapp-status-reactions"); -export const runWhatsAppStatusReactionLifecycleScenario = ( - context: WhatsAppQaScenarioEnvironment, -) => runWhatsAppScenario(context, "whatsapp-status-reaction-lifecycle"); -export const runWhatsAppGroupAllowlistBlockScenario = (context: WhatsAppQaScenarioEnvironment) => - runWhatsAppScenario(context, "whatsapp-group-allowlist-block"); -export const runWhatsAppApprovalExecNativeScenario = (context: WhatsAppQaScenarioEnvironment) => - runWhatsAppScenario(context, "whatsapp-approval-exec-native"); -export const runWhatsAppApprovalExecReactionNativeScenario = ( - context: WhatsAppQaScenarioEnvironment, -) => runWhatsAppScenario(context, "whatsapp-approval-exec-reaction-native"); -export const runWhatsAppApprovalExecGroupReactionNativeScenario = ( - context: WhatsAppQaScenarioEnvironment, -) => runWhatsAppScenario(context, "whatsapp-approval-exec-group-reaction-native"); -export const runWhatsAppApprovalPluginNativeScenario = (context: WhatsAppQaScenarioEnvironment) => - runWhatsAppScenario(context, "whatsapp-approval-plugin-native"); diff --git a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.approvals.ts b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.approvals.ts index 20e96f5cf881..9296f7abad32 100644 --- a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.approvals.ts +++ b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.approvals.ts @@ -16,7 +16,7 @@ import type { WhatsAppQaApprovalKind, WhatsAppQaApprovalScenarioRun, WhatsAppQaGateway, - WhatsAppQaScenarioDefinition, + WhatsAppQaScenarioMetadata, } from "./whatsapp-live.contracts.js"; import { formatDiagnosticId } from "./whatsapp-live.operations.js"; @@ -210,7 +210,7 @@ async function waitForWhatsAppApprovalMessage(params: { driver: WhatsAppQaDriverSession; observedAfter?: Date; observedMessages: WhatsAppObservedMessage[]; - scenario: WhatsAppQaScenarioDefinition; + scenario: WhatsAppQaScenarioMetadata; state: "pending" | "resolved"; sutPhoneE164: string; timeoutMs: number; @@ -267,7 +267,7 @@ export async function runWhatsAppApprovalScenario(params: { gateway: WhatsAppQaGateway; observedMessages: WhatsAppObservedMessage[]; run: WhatsAppQaApprovalScenarioRun; - scenario: WhatsAppQaScenarioDefinition; + scenario: WhatsAppQaScenarioMetadata; sutAccountId: string; sutPhoneE164: string; turnSourceTo: string; diff --git a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.contracts.ts b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.contracts.ts index 3978f1fe9aba..cd7020a8aae0 100644 --- a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.contracts.ts +++ b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.contracts.ts @@ -14,46 +14,6 @@ export type WhatsAppQaRuntimeEnv = { groupJid?: string; }; -export type WhatsAppQaScenarioId = - | "whatsapp-approval-exec-deny-native" - | "whatsapp-approval-exec-group-reaction-native" - | "whatsapp-approval-exec-reaction-native" - | "whatsapp-agent-message-action-react" - | "whatsapp-agent-message-action-upload-file" - | "whatsapp-audio-preflight" - | "whatsapp-broadcast-group-fanout" - | "whatsapp-canary" - | "whatsapp-group-allowlist-block" - | "whatsapp-group-activation-always" - | "whatsapp-group-agent-message-action-react" - | "whatsapp-group-agent-message-action-upload-file" - | "whatsapp-group-audio-gating" - | "whatsapp-group-outbound-audio" - | "whatsapp-group-outbound-media" - | "whatsapp-group-outbound-poll" - | "whatsapp-group-pending-history-context" - | "whatsapp-group-reply-to-bot-triggers" - | "whatsapp-group-reply-to-message" - | "whatsapp-inbound-reaction-no-trigger" - | "whatsapp-inbound-image-caption" - | "whatsapp-inbound-structured-messages" - | "whatsapp-message-actions" - | "whatsapp-outbound-document-preserves-filename" - | "whatsapp-outbound-media-matrix" - | "whatsapp-outbound-poll" - | "whatsapp-outbound-send-serialization" - | "whatsapp-mention-gating" - | "whatsapp-reply-delivery-shape" - | "whatsapp-reply-context-isolation" - | "whatsapp-reply-to-message" - | "whatsapp-reply-to-mode-batched" - | "whatsapp-stream-final-message-accounting" - | "whatsapp-status-reaction-lifecycle" - | "whatsapp-status-reactions" - | "whatsapp-top-level-reply-shape" - | "whatsapp-approval-exec-native" - | "whatsapp-approval-plugin-native"; - export type WhatsAppQaApprovalKind = "exec" | "plugin"; export type WhatsAppQaApprovalDecision = "allow-once" | "deny"; type WhatsAppQaApprovalDecisionMode = "reaction" | "rpc"; @@ -63,47 +23,6 @@ export function toWhatsAppQaError(error: unknown): Error { return error instanceof Error ? error : new Error(formatErrorMessage(error)); } -const WHATSAPP_QA_SCENARIO_POSTURES = { - "whatsapp-agent-message-action-react": "user-path", - "whatsapp-agent-message-action-upload-file": "user-path", - "whatsapp-approval-exec-deny-native": "native-approval", - "whatsapp-approval-exec-group-reaction-native": "native-approval", - "whatsapp-approval-exec-native": "native-approval", - "whatsapp-approval-exec-reaction-native": "native-approval", - "whatsapp-approval-plugin-native": "native-approval", - "whatsapp-audio-preflight": "user-path", - "whatsapp-broadcast-group-fanout": "user-path", - "whatsapp-canary": "user-path", - "whatsapp-group-activation-always": "user-path", - "whatsapp-group-allowlist-block": "user-path", - "whatsapp-group-agent-message-action-react": "user-path", - "whatsapp-group-agent-message-action-upload-file": "user-path", - "whatsapp-group-audio-gating": "user-path", - "whatsapp-group-outbound-audio": "direct-gateway", - "whatsapp-group-outbound-media": "direct-gateway", - "whatsapp-group-outbound-poll": "direct-gateway", - "whatsapp-group-pending-history-context": "user-path", - "whatsapp-group-reply-to-bot-triggers": "user-path", - "whatsapp-group-reply-to-message": "user-path", - "whatsapp-inbound-image-caption": "user-path", - "whatsapp-inbound-reaction-no-trigger": "user-path", - "whatsapp-inbound-structured-messages": "user-path", - "whatsapp-mention-gating": "user-path", - "whatsapp-message-actions": "direct-gateway", - "whatsapp-outbound-document-preserves-filename": "direct-gateway", - "whatsapp-outbound-media-matrix": "direct-gateway", - "whatsapp-outbound-poll": "direct-gateway", - "whatsapp-outbound-send-serialization": "direct-gateway", - "whatsapp-reply-context-isolation": "direct-gateway", - "whatsapp-reply-delivery-shape": "direct-gateway", - "whatsapp-reply-to-message": "user-path", - "whatsapp-reply-to-mode-batched": "user-path", - "whatsapp-status-reaction-lifecycle": "user-path", - "whatsapp-status-reactions": "user-path", - "whatsapp-stream-final-message-accounting": "user-path", - "whatsapp-top-level-reply-shape": "user-path", -} satisfies Record; - type WhatsAppQaMessageSendMode = | { kind?: "text"; @@ -124,7 +43,7 @@ export type WhatsAppQaGatewayRuntime = Pick< export type WhatsAppQaGatewayCallContext = { gateway: Pick; gatewayTarget: string; - scenarioId: WhatsAppQaScenarioId; + scenarioId: string; sutAccountId: string; }; export type WhatsAppQaObservedMessagesContext = { @@ -145,7 +64,7 @@ export type WhatsAppQaMessageScenarioContext = { gatewayWorkspaceDir: string; recordObservedMessage: (message: WhatsAppQaDriverObservedMessage) => void; requestStartedAt: Date; - scenarioId: WhatsAppQaScenarioId; + scenarioId: string; scenarioTitle: string; sent: { messageId?: string }; sutAccountId: string; @@ -166,7 +85,7 @@ type WhatsAppQaResolvedScenarioTarget = export function resolveWhatsAppQaScenarioTarget(params: { groupJid?: string; - scenarioId: WhatsAppQaScenarioId; + scenarioId: string; target: "dm" | "group"; }): WhatsAppQaResolvedScenarioTarget { if (params.target === "dm") { @@ -242,7 +161,7 @@ export type WhatsAppQaApprovalScenarioRun = { token: string; }; -type WhatsAppQaScenarioRun = WhatsAppQaApprovalScenarioRun | WhatsAppQaMessageScenarioRun; +export type WhatsAppQaScenarioRun = WhatsAppQaApprovalScenarioRun | WhatsAppQaMessageScenarioRun; export type WhatsAppQaConfigOverrides = { actions?: boolean; @@ -263,14 +182,17 @@ export type WhatsAppQaConfigOverrides = { statusReactions?: boolean; }; -export type WhatsAppQaScenarioDefinition = { - id: WhatsAppQaScenarioId; - title: string; - timeoutMs: number; +export type WhatsAppQaScenarioImplementation = { buildRun: () => WhatsAppQaScenarioRun; configOverrides?: WhatsAppQaConfigOverrides; + posture: WhatsAppQaScenarioPosture; requiresGroupJid?: boolean; - requiredPluginIds?: readonly string[]; +}; + +export type WhatsAppQaScenarioMetadata = { + id: string; + timeoutMs: number; + title: string; }; export interface WhatsAppObservedMessage extends WhatsAppQaDriverObservedMessage { @@ -297,10 +219,13 @@ export type WhatsAppQaScenarioResult = { title: string; }; -export function buildWhatsAppQaScenarioResultBase(scenario: WhatsAppQaScenarioDefinition) { +export function buildWhatsAppQaScenarioResultBase( + scenario: WhatsAppQaScenarioMetadata, + implementation: WhatsAppQaScenarioImplementation, +) { return { id: scenario.id, title: scenario.title, - posture: WHATSAPP_QA_SCENARIO_POSTURES[scenario.id], + posture: implementation.posture, }; } diff --git a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.gateway.ts b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.gateway.ts index 52c751c7f778..338430da1294 100644 --- a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.gateway.ts +++ b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.gateway.ts @@ -7,10 +7,9 @@ import type { WhatsAppQaGatewayCallContext, WhatsAppQaGatewayRuntime, WhatsAppQaMessageScenarioContext, - WhatsAppQaScenarioId, } from "./whatsapp-live.contracts.js"; -function buildWhatsAppQaIdempotencyKey(scenarioId: WhatsAppQaScenarioId, label: string) { +function buildWhatsAppQaIdempotencyKey(scenarioId: string, label: string) { return `${scenarioId}:${label}:${randomUUID()}`; } diff --git a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.test.ts b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.test.ts index 07002f4053b8..f00498acd049 100644 --- a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.test.ts @@ -13,12 +13,18 @@ import { import { describe, expect, it, vi } from "vitest"; import { fingerprintQaCredentialId } from "../../qa-credentials-fingerprint.runtime.js"; import { readQaScenarioById } from "../../scenario-catalog.js"; +import { requireFlowScenario } from "../../scenario-catalog.test-utils.js"; import { applyQaMergePatch, collectQaSuiteGatewayConfigPatch } from "../../suite-planning.js"; import { createWhatsAppQaScenarioEnvironment } from "./scenario-environment.js"; import { resolveWhatsAppQaScenarioIds } from "./scenario-selection.js"; import { runWhatsAppApprovalScenario } from "./whatsapp-live.approvals.js"; import { buildWhatsAppQaConfig, parseWhatsAppQaCredentialPayload } from "./whatsapp-live.config.js"; -import { resolveWhatsAppQaMessageTargets } from "./whatsapp-live.contracts.js"; +import { + resolveWhatsAppQaMessageTargets, + type WhatsAppQaScenarioImplementation, + type WhatsAppQaScenarioMetadata, + type WhatsAppQaScenarioRun, +} from "./whatsapp-live.contracts.js"; import { callWhatsAppGatewayMessageAction, callWhatsAppGatewayPoll, @@ -27,7 +33,10 @@ import { runWhatsAppStructuredInboundChecks, waitForScenarioObservedMessage, } from "./whatsapp-live.operations.js"; -import { getWhatsAppQaScenarioDefinition } from "./whatsapp-live.scenarios.js"; +import * as whatsappCapabilityScenarios from "./whatsapp-live.scenario-implementations.capabilities.js"; +import * as whatsappConversationScenarios from "./whatsapp-live.scenario-implementations.conversation.js"; +import * as whatsappDeliveryScenarios from "./whatsapp-live.scenario-implementations.delivery.js"; +import * as whatsappUserPathScenarios from "./whatsapp-live.scenario-implementations.user-path.js"; import { unpackWhatsAppAuthArchive } from "./whatsapp-live.setup.js"; const runExecSpy = vi.hoisted(() => @@ -108,9 +117,8 @@ function createWhatsAppQaDriverMock( }; } -type WhatsAppScenarioDefinition = ReturnType; -type WhatsAppScenarioRun = ReturnType; -type WhatsAppMessageScenarioRun = Exclude; +type WhatsAppScenarioDefinition = WhatsAppQaScenarioMetadata & WhatsAppQaScenarioImplementation; +type WhatsAppMessageScenarioRun = Exclude; type WhatsAppScenarioContext = Parameters>[0]; type WhatsAppQaConfigBase = Parameters[0]; type WhatsAppQaConfigParams = Parameters[1]; @@ -159,14 +167,44 @@ function buildWhatsAppQaConfigFixture( }); } -type WhatsAppScenarioIdFilter = Parameters[0]; +type WhatsAppScenarioIdFilter = string; + +const whatsappScenarioImplementations = { + ...whatsappCapabilityScenarios, + ...whatsappConversationScenarios, + ...whatsappDeliveryScenarios, + ...whatsappUserPathScenarios, +} as Record; + +function toWhatsAppScenarioExportName(id: string) { + const suffix = id + .slice("whatsapp-".length) + .split("-") + .map((part) => `${part.slice(0, 1).toUpperCase()}${part.slice(1)}`) + .join(""); + return `whatsappQa${suffix}Scenario`; +} + +function getWhatsAppScenario(id: string): WhatsAppScenarioDefinition { + const implementation = whatsappScenarioImplementations[toWhatsAppScenarioExportName(id)]; + if (!implementation) { + throw new Error(`missing WhatsApp test implementation for ${id}`); + } + const scenario = requireFlowScenario(readQaScenarioById(id)); + return { + ...implementation, + id, + timeoutMs: scenario.execution.timeoutMs ?? 60_000, + title: scenario.title, + }; +} function findScenarios(ids: readonly string[]) { - return ids.map((id) => getWhatsAppQaScenarioDefinition(id)); + return ids.map(getWhatsAppScenario); } function findWhatsAppScenario(id: WhatsAppScenarioIdFilter) { - return getWhatsAppQaScenarioDefinition(id); + return getWhatsAppScenario(id); } function updateObservedMessage( @@ -202,7 +240,7 @@ const WHATSAPP_GROUP_CAPABILITY_SCENARIO_IDS = [ ] as const satisfies readonly WhatsAppScenarioIdFilter[]; function findMockWhatsAppScenario(id: WhatsAppScenarioIdFilter) { - const scenario = getWhatsAppQaScenarioDefinition(id); + const scenario = getWhatsAppScenario(id); const mockScenarioIds = new Set(resolveWhatsAppQaScenarioIds({ providerMode: "mock-openai" })); if (!mockScenarioIds.has(id)) { throw new Error(`missing WhatsApp mock-openai scenario ${id}`); @@ -1114,14 +1152,19 @@ describe("WhatsApp QA live runtime", () => { sutAuthDir: "/tmp/whatsapp-sut", }); - await prepareFlow({ - config: { whatsappScenarioId: "whatsapp-canary" }, + const prepared = await prepareFlow({ + config: {}, gateway: { call: gatewayCall } as never, outputDir: "/tmp/whatsapp-output", primaryModel: "mock-openai/gpt-5.6-luna", + scenarioId: "whatsapp-canary", + scenarioTitle: "WhatsApp DM canary", timeoutMs: 60_000, waitForConfigRestartSettle: vi.fn(), }); + await prepared.whatsappScenarioContext.configureScenario( + findWhatsAppScenario("whatsapp-canary"), + ); const patchCall = gatewayCall.mock.calls.find(([method]) => method === "config.patch"); if (!patchCall) { @@ -1152,16 +1195,19 @@ describe("WhatsApp QA live runtime", () => { sutAuthDir: "/tmp/whatsapp-sut", }); - await expect( - prepareFlow({ - config: { policyKey: "dmPolicy", policyValue: "disabled" }, - gateway: { call: gatewayCall } as never, - outputDir: "/tmp/whatsapp-output", - primaryModel: "mock-openai/gpt-5.6-luna", - timeoutMs: 60_000, - waitForConfigRestartSettle: vi.fn(), - }), - ).resolves.toBeUndefined(); + const prepared = await prepareFlow({ + config: { policyKey: "dmPolicy", policyValue: "disabled" }, + gateway: { call: gatewayCall } as never, + outputDir: "/tmp/whatsapp-output", + primaryModel: "mock-openai/gpt-5.6-luna", + scenarioId: "whatsapp-access-control-dm-disabled", + scenarioTitle: "WhatsApp dmPolicy disabled stays quiet", + timeoutMs: 60_000, + waitForConfigRestartSettle: vi.fn(), + }); + expect(prepared.whatsappScenarioContext.scenario.id).toBe( + "whatsapp-access-control-dm-disabled", + ); expect(gatewayCall).not.toHaveBeenCalled(); }); @@ -2084,7 +2130,7 @@ describe("WhatsApp QA live runtime", () => { throw new Error("whatsapp-audio-preflight unexpectedly built an approval scenario run"); } - expect(scenario.requiredPluginIds).toEqual(["openai"]); + expect(readQaScenarioById(scenario.id).plugins).toEqual(["openai"]); expect(scenarioRun.expectReply).toBe(true); expect(scenarioRun.matchText).toBe("WHATSAPP_QA_AUDIO_TRANSCRIPT_OK"); expect(scenarioRun.sendMode).toMatchObject({ diff --git a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenario-implementations.capabilities.ts b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenario-implementations.capabilities.ts new file mode 100644 index 000000000000..aaa45b5586dd --- /dev/null +++ b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenario-implementations.capabilities.ts @@ -0,0 +1,408 @@ +// QA Lab WhatsApp Gateway capability and structured-message scenarios. +import { randomUUID } from "node:crypto"; +import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; +import type { WhatsAppQaScenarioImplementation } from "./whatsapp-live.contracts.js"; +import { + WHATSAPP_QA_AUDIO_OGG_OPUS_MIME, + WHATSAPP_QA_GROUP_AUDIO_TRANSCRIPT_MARKER, + WHATSAPP_QA_ONE_PIXEL_PNG, + callWhatsAppGatewayMessageAction, + callWhatsAppGatewayPoll, + callWhatsAppGatewaySend, + callWhatsAppGatewaySendConcurrently, + createWhatsAppQaAudioOggOpusBuffer, + createWhatsAppQaPdfBuffer, + requireWhatsAppTriggerMessageId, + runWhatsAppStructuredInboundChecks, + waitForScenarioObservedMessage, + waitForWhatsAppScenarioSutMessage, + waitForWhatsAppSutReactionToTrigger, + writeWhatsAppQaWorkspaceFixture, +} from "./whatsapp-live.operations.js"; + +export const whatsappQaOutboundDocumentPreservesFilenameScenario: WhatsAppQaScenarioImplementation = + { + posture: "direct-gateway", + buildRun: () => { + const token = `WHATSAPP_QA_DOCUMENT_FILE_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + afterReply: async (_reply, context) => { + const documentPath = await writeWhatsAppQaWorkspaceFixture(context, { + buffer: createWhatsAppQaPdfBuffer(), + fileName: `whatsapp-qa-report-${token}.pdf`, + }); + const documentStartedAt = new Date(); + await callWhatsAppGatewaySend(context, { + forceDocument: true, + label: "document-filename", + mediaUrl: documentPath, + message: `${token}_CAPTION`, + }); + const document = await waitForScenarioObservedMessage(context, { + observedAfter: documentStartedAt, + match: (message) => + message.kind === "media" && + message.hasMedia === true && + message.text.includes(`${token}_CAPTION`) && + message.mediaFileName === `whatsapp-qa-report-${token}.pdf`, + }); + return `document ${document.mediaFileName ?? ""} preserved`; + }, + configMode: "allowlist", + expectReply: true, + input: `Reply with only this exact marker before document filename check: ${token}`, + matchText: token, + target: "dm", + }; + }, + }; + +export const whatsappQaOutboundSendSerializationScenario: WhatsAppQaScenarioImplementation = { + posture: "direct-gateway", + buildRun: () => { + const token = `WHATSAPP_QA_SERIAL_SEND_${randomUUID().slice(0, 8).toUpperCase()}`; + const markers = Array.from({ length: 5 }, (_, index) => `${token}_${index + 1}`); + return { + afterReply: async (_reply, context) => { + const sendsStartedAt = new Date(); + await callWhatsAppGatewaySendConcurrently( + context, + markers.map((marker, index) => ({ + label: `parallel-${index + 1}`, + message: marker, + })), + ); + await Promise.all( + markers.map((marker) => + waitForScenarioObservedMessage(context, { + observedAfter: sendsStartedAt, + match: (message) => message.kind === "text" && message.text.includes(marker), + }), + ), + ); + return `gateway parallel send delivered ${markers.length}/${markers.length} messages`; + }, + configMode: "allowlist", + expectReply: true, + input: `Reply with only this exact marker before parallel send checks: ${token}`, + matchText: token, + target: "dm", + }; + }, +}; + +export const whatsappQaOutboundPollScenario: WhatsAppQaScenarioImplementation = { + posture: "direct-gateway", + buildRun: () => { + const token = `WHATSAPP_QA_OUTBOUND_POLL_${randomUUID().slice(0, 8).toUpperCase()}`; + const question = `${token} choose one`; + return { + afterReply: async (_reply, context) => { + const pollStartedAt = new Date(); + await callWhatsAppGatewayPoll(context, { + label: "poll", + options: ["alpha", "beta"], + question, + }); + const poll = await waitForScenarioObservedMessage(context, { + observedAfter: pollStartedAt, + match: (message) => + message.kind === "poll" && + message.poll?.question === question && + message.poll.options.includes("alpha") && + message.poll.options.includes("beta"), + }); + return `poll observed with ${poll.poll?.options.length ?? 0} options`; + }, + configMode: "allowlist", + expectReply: true, + input: `Reply with only this exact marker before outbound poll check: ${token}`, + matchText: token, + target: "dm", + }; + }, +}; + +export const whatsappQaGroupOutboundMediaScenario: WhatsAppQaScenarioImplementation = { + posture: "direct-gateway", + requiresGroupJid: true, + buildRun: () => { + const token = `WHATSAPP_QA_GROUP_OUTBOUND_MEDIA_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + afterReply: async (_reply, context) => { + const mediaRootToken = randomUUID().slice(0, 8); + const imagePath = await writeWhatsAppQaWorkspaceFixture(context, { + buffer: WHATSAPP_QA_ONE_PIXEL_PNG, + fileName: `whatsapp-qa-group-${mediaRootToken}.png`, + }); + const documentPath = await writeWhatsAppQaWorkspaceFixture(context, { + buffer: createWhatsAppQaPdfBuffer(), + fileName: `whatsapp-qa-group-${mediaRootToken}.pdf`, + }); + + const imageStartedAt = new Date(); + await callWhatsAppGatewaySend(context, { + label: "group-image", + mediaUrl: imagePath, + message: `${token}_IMAGE`, + }); + await waitForWhatsAppScenarioSutMessage(context, { + observedAfter: imageStartedAt, + targetKind: "group", + match: (message) => + message.kind === "media" && + message.hasMedia === true && + message.mediaType?.startsWith("image/") === true && + message.text.includes(`${token}_IMAGE`), + }); + + const documentStartedAt = new Date(); + await callWhatsAppGatewaySend(context, { + forceDocument: true, + label: "group-document", + mediaUrl: documentPath, + message: `${token}_DOCUMENT`, + }); + await waitForWhatsAppScenarioSutMessage(context, { + observedAfter: documentStartedAt, + targetKind: "group", + match: (message) => + message.kind === "media" && + message.hasMedia === true && + (message.mediaType === "application/pdf" || + message.mediaFileName?.endsWith(".pdf") === true) && + message.text.includes(`${token}_DOCUMENT`), + }); + return "gateway send delivered image and document media to the group"; + }, + configMode: "allowlist", + expectReply: true, + input: `openclawqa reply with only this exact marker before group outbound media checks: ${token}`, + matchText: token, + target: "group", + }; + }, +}; + +export const whatsappQaGroupOutboundAudioScenario: WhatsAppQaScenarioImplementation = { + posture: "direct-gateway", + requiresGroupJid: true, + buildRun: () => { + const token = `WHATSAPP_QA_GROUP_OUTBOUND_AUDIO_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + afterReply: async (_reply, context) => { + const audioPath = await writeWhatsAppQaWorkspaceFixture(context, { + buffer: createWhatsAppQaAudioOggOpusBuffer({ variant: "group-trigger" }), + fileName: `whatsapp-qa-group-audio-${token}.ogg`, + }); + const audioStartedAt = new Date(); + await callWhatsAppGatewaySend(context, { + asVoice: true, + label: "group-audio", + mediaUrl: audioPath, + message: `${token}_AUDIO`, + }); + await waitForWhatsAppScenarioSutMessage(context, { + observedAfter: audioStartedAt, + targetKind: "group", + match: (message) => + message.kind === "media" && + message.hasMedia === true && + message.mediaType?.startsWith("audio/") === true, + }); + await waitForWhatsAppScenarioSutMessage(context, { + observedAfter: audioStartedAt, + targetKind: "group", + match: (message) => message.text.includes(`${token}_AUDIO`), + }); + return "gateway send delivered audio media to the group"; + }, + configMode: "allowlist", + expectReply: true, + input: `openclawqa reply with only this exact marker before group outbound audio check: ${token}`, + matchText: token, + target: "group", + }; + }, +}; + +export const whatsappQaGroupOutboundPollScenario: WhatsAppQaScenarioImplementation = { + posture: "direct-gateway", + requiresGroupJid: true, + buildRun: () => { + const token = `WHATSAPP_QA_GROUP_OUTBOUND_POLL_${randomUUID().slice(0, 8).toUpperCase()}`; + const question = `${token} choose one`; + return { + afterReply: async (_reply, context) => { + const pollStartedAt = new Date(); + await callWhatsAppGatewayPoll(context, { + label: "group-poll", + options: ["alpha", "beta"], + question, + }); + const poll = await waitForWhatsAppScenarioSutMessage(context, { + observedAfter: pollStartedAt, + targetKind: "group", + match: (message) => + message.kind === "poll" && + message.poll?.question === question && + message.poll.options.includes("alpha") && + message.poll.options.includes("beta"), + }); + return `group poll observed with ${poll.poll?.options.length ?? 0} options`; + }, + configMode: "allowlist", + expectReply: true, + input: `openclawqa reply with only this exact marker before group outbound poll check: ${token}`, + matchText: token, + target: "group", + }; + }, +}; + +export const whatsappQaMessageActionsScenario: WhatsAppQaScenarioImplementation = { + posture: "direct-gateway", + configOverrides: { + actions: true, + }, + buildRun: () => { + const token = `WHATSAPP_QA_ACTIONS_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + afterReply: async (_reply, context) => { + const triggerMessageId = requireWhatsAppTriggerMessageId(context); + const reactionStartedAt = new Date(); + await callWhatsAppGatewayMessageAction(context, { + action: "react", + label: "react", + params: { + emoji: "👍", + messageId: triggerMessageId, + }, + }); + await waitForWhatsAppSutReactionToTrigger(context, { + expectation: { emoji: "👍" }, + observedAfter: reactionStartedAt, + }); + + const uploadStartedAt = new Date(); + await callWhatsAppGatewayMessageAction(context, { + action: "upload-file", + label: "upload-file", + params: { + buffer: WHATSAPP_QA_ONE_PIXEL_PNG.toString("base64"), + caption: `${token}_UPLOAD`, + contentType: "image/png", + filename: "whatsapp-qa-upload.png", + }, + }); + await waitForScenarioObservedMessage(context, { + observedAfter: uploadStartedAt, + match: (message) => + message.kind === "media" && + message.mediaType?.startsWith("image/") === true && + message.text.includes(`${token}_UPLOAD`), + }); + return "message.action react and upload-file observed"; + }, + configMode: "allowlist", + expectReply: true, + input: `Reply with only this exact marker before action checks: ${token}`, + matchText: token, + target: "dm", + }; + }, +}; + +export const whatsappQaInboundStructuredMessagesScenario: WhatsAppQaScenarioImplementation = { + posture: "user-path", + buildRun: () => { + const token = `WHATSAPP_QA_STRUCTURED_${randomUUID().slice(0, 8).toUpperCase()}`; + const locationToken = `${token}_LOCATION`; + const contactToken = `${token}_CONTACT`; + const stickerToken = `${token}_STICKER`; + const locationCoordinateText = "37.774900, -122.419400"; + return { + afterReply: async (_reply, context) => { + const waitForStructuredReply = async ( + label: string, + observedAfter: Date, + expectedToken: string, + ) => { + try { + return await waitForScenarioObservedMessage(context, { + observedAfter, + timeoutMs: 60_000, + match: (message) => message.text.includes(expectedToken), + diagnosticChecks: [ + { + label: "containsExpectedToken", + match: (message) => message.text.includes(expectedToken), + }, + ], + }); + } catch (error) { + throw new Error( + `timed out waiting for WhatsApp structured ${label} reply (${expectedToken}): ${formatErrorMessage(error)}`, + { cause: error }, + ); + } + }; + + await runWhatsAppStructuredInboundChecks({ + contactToken, + documentToken: `${token}_DOCUMENT`, + driver: context.driver, + driverPhoneE164: context.driverPhoneE164, + locationToken, + stickerToken, + target: context.target, + waitForStructuredReply, + }); + return "document, location, contact, and sticker elicited replies"; + }, + configMode: "allowlist", + expectReply: true, + input: + `When a later WhatsApp location message shows ${locationCoordinateText}, ` + + `reply with only this WhatsApp location marker: ${locationToken}. ` + + `When a later WhatsApp contact message appears, ` + + `reply with only this WhatsApp contact marker: ${contactToken}. ` + + `When a later WhatsApp sticker message appears, ` + + `reply with only this WhatsApp sticker marker: ${stickerToken}. ` + + `Reply with only this exact marker before structured inbound checks: ${token}`, + matchText: token, + target: "dm", + }; + }, +}; + +export const whatsappQaGroupAudioGatingScenario: WhatsAppQaScenarioImplementation = { + posture: "user-path", + configOverrides: { + audioPreflight: true, + }, + requiresGroupJid: true, + buildRun: () => ({ + configMode: "allowlist", + expectReply: true, + input: "", + matchText: WHATSAPP_QA_GROUP_AUDIO_TRANSCRIPT_MARKER, + quietInput: "", + quietSendMode: { + fileName: "whatsapp-qa-group-audio-quiet.ogg", + kind: "media", + mediaBuffer: createWhatsAppQaAudioOggOpusBuffer(), + mediaType: WHATSAPP_QA_AUDIO_OGG_OPUS_MIME, + }, + quietWindowMs: 5_000, + sendMode: { + fileName: "whatsapp-qa-group-audio.ogg", + kind: "media", + mediaBuffer: createWhatsAppQaAudioOggOpusBuffer({ + variant: "group-trigger", + }), + mediaType: WHATSAPP_QA_AUDIO_OGG_OPUS_MIME, + }, + target: "group", + }), +}; diff --git a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenario-implementations.conversation.ts b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenario-implementations.conversation.ts new file mode 100644 index 000000000000..decec09eccb9 --- /dev/null +++ b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenario-implementations.conversation.ts @@ -0,0 +1,366 @@ +// QA Lab WhatsApp conversation and reply-context scenarios. +import { randomUUID } from "node:crypto"; +import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; +import { + toWhatsAppQaError, + type WhatsAppQaMessageScenarioRun, + type WhatsAppQaScenarioImplementation, +} from "./whatsapp-live.contracts.js"; +import { + assertWhatsAppMessageFromSutPhone, + assertWhatsAppMessagesFromSutPhone, + buildWhatsAppQuotedMessageKeyFromObservedMessage, + resolveWhatsAppQaNoReplyTarget, + waitForDistinctWhatsAppSutMessages, + waitForNoWhatsAppReply, + waitForWhatsAppScenarioSutMessage, +} from "./whatsapp-live.operations.js"; + +function buildWhatsAppQuoteReplyRun(target: "dm" | "group"): WhatsAppQaMessageScenarioRun { + const token = `WHATSAPP_QA_REPLY_TO_${target.toUpperCase()}_${randomUUID().slice(0, 8).toUpperCase()}`; + const input = + target === "group" + ? `openclawqa reply with only this exact marker: ${token}` + : `Reply with only this exact marker: ${token}`; + return { + configMode: "allowlist", + expectReply: true, + input, + matchText: token, + target, + verify: (reply, context) => { + if (!context.sent.messageId) { + throw new Error("WhatsApp driver did not return a triggering message id."); + } + if (reply.quoted?.messageId !== context.sent.messageId) { + throw new Error( + `expected reply quote ${context.sent.messageId}, got ${reply.quoted?.messageId ?? ""}`, + ); + } + }, + }; +} + +export const whatsappQaCanaryScenario: WhatsAppQaScenarioImplementation = { + posture: "user-path", + buildRun: () => { + const token = `WHATSAPP_QA_ECHO_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + configMode: "allowlist", + expectReply: true, + input: `Reply with only this exact marker: ${token}`, + matchText: token, + target: "dm", + }; + }, +}; + +export const whatsappQaMentionGatingScenario: WhatsAppQaScenarioImplementation = { + posture: "user-path", + requiresGroupJid: true, + buildRun: () => { + const quietToken = `WHATSAPP_QA_GROUP_QUIET_${randomUUID().slice(0, 8).toUpperCase()}`; + const replyToken = `WHATSAPP_QA_GROUP_MENTION_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + configMode: "allowlist", + expectReply: true, + input: `openclawqa reply with only this exact marker: ${replyToken}`, + matchText: replyToken, + quietInput: `This group message is intentionally unmentioned. If you respond, include ${quietToken}.`, + quietMatchText: quietToken, + quietWindowMs: 5_000, + target: "group", + }; + }, +}; + +export const whatsappQaGroupPendingHistoryContextScenario: WhatsAppQaScenarioImplementation = { + posture: "user-path", + configOverrides: { + groupHistoryLimit: 50, + groupPolicy: "open", + inboundDebounceMs: 0, + replyToMode: "all", + }, + requiresGroupJid: true, + buildRun: () => { + const suffix = randomUUID().slice(0, 8).toUpperCase(); + const quietMarker = `WHATSAPP_QA_PENDING_HISTORY_QUIET_${suffix}`; + const contextSentinel = `WHATSAPP_QA_PENDING_HISTORY_CONTEXT_ONLY_${suffix}`; + const triggerMarker = `WHATSAPP_QA_PENDING_HISTORY_TRIGGER_${suffix}`; + const okMarker = `WHATSAPP_QA_PENDING_HISTORY_OK_${suffix}`; + return { + configMode: "open", + expectReply: true, + expectedSutMessageCount: 1, + input: + `openclawqa pending history context check ${triggerMarker}. ` + + `Reply with only ${okMarker} only if the previous quiet group message is present ` + + `in prior group context with its context-only sentinel. ` + + "Do not use current-message text as proof.", + matchText: okMarker, + quietInput: `quiet context marker ${quietMarker} ${contextSentinel}`, + quietWindowMs: 5_000, + target: "group", + }; + }, +}; + +export const whatsappQaBroadcastGroupFanoutScenario: WhatsAppQaScenarioImplementation = { + posture: "user-path", + configOverrides: { + broadcast: { + agents: ["main", "qa-second"], + strategy: "sequential", + }, + groupPolicy: "open", + }, + requiresGroupJid: true, + buildRun: () => { + const token = `WHATSAPP_QA_BROADCAST_TOKEN_${randomUUID().slice(0, 8).toUpperCase()}`; + const mainMarker = `${token}_MAIN`; + const secondMarker = `${token}_SECOND`; + return { + afterReply: async (reply, context) => { + const replies = await waitForDistinctWhatsAppSutMessages(context, { + initialMessages: [reply], + matchers: [ + (message) => message.text.includes(mainMarker), + (message) => message.text.includes(secondMarker), + ], + observedAfter: context.requestStartedAt, + timeoutMs: 60_000, + }); + assertWhatsAppMessagesFromSutPhone(replies, context); + return "broadcast fanout produced main and qa-second replies"; + }, + configMode: "open", + expectReply: true, + input: `openclawqa broadcast fanout check ${token}`, + matchText: mainMarker, + target: "group", + }; + }, +}; + +export const whatsappQaGroupActivationAlwaysScenario: WhatsAppQaScenarioImplementation = { + posture: "user-path", + configOverrides: { + groupPolicy: "open", + }, + requiresGroupJid: true, + buildRun: () => { + const suffix = randomUUID().slice(0, 8).toUpperCase(); + const alwaysMarker = `WHATSAPP_QA_ACTIVATION_ALWAYS_${suffix}`; + const quietMarker = `WHATSAPP_QA_ACTIVATION_QUIET_${suffix}`; + return { + afterReply: async (reply, context) => { + assertWhatsAppMessageFromSutPhone(reply, context); + let activationProbeError: unknown; + try { + const alwaysStartedAt = new Date(); + await context.driver.sendText( + context.target, + `Group activation visible behavior marker ${alwaysMarker}`, + ); + const alwaysReply = await waitForWhatsAppScenarioSutMessage(context, { + match: (message) => message.text.includes(alwaysMarker), + observedAfter: alwaysStartedAt, + targetKind: "group", + timeoutMs: 60_000, + }); + assertWhatsAppMessageFromSutPhone(alwaysReply, context); + } catch (error) { + activationProbeError = error; + } + + let restoreError: unknown; + const restoreStartedAt = new Date(); + try { + await context.driver.sendText(context.target, "/activation mention"); + const restoreReply = await waitForWhatsAppScenarioSutMessage(context, { + match: (message) => /\bactivation\b.*\bmention\b/iu.test(message.text), + observedAfter: restoreStartedAt, + targetKind: "group", + timeoutMs: 60_000, + }); + assertWhatsAppMessageFromSutPhone(restoreReply, context); + } catch (error) { + restoreError = error; + } + + if (activationProbeError && restoreError) { + throw new Error( + `activation always probe failed; additionally failed to restore mention mode: ${formatErrorMessage(restoreError)}`, + { cause: activationProbeError }, + ); + } + if (activationProbeError) { + throw toWhatsAppQaError(activationProbeError); + } + if (restoreError) { + throw toWhatsAppQaError(restoreError); + } + + const quietStartedAt = new Date(); + await context.driver.sendText( + context.target, + `Group activation quiet marker ${quietMarker}`, + ); + await waitForNoWhatsAppReply({ + driver: context.driver, + observedAfter: quietStartedAt, + sutPhoneE164: context.sutPhoneE164, + windowMs: 5_000, + ...resolveWhatsAppQaNoReplyTarget({ + groupJid: context.target, + target: "group", + }), + }); + return "activation always replied to an unmentioned group message and mention mode was restored"; + }, + configMode: "allowlist", + expectReply: true, + input: "/activation always", + matchText: /\bactivation\b.*\balways\b/iu, + target: "group", + }; + }, +}; + +export const whatsappQaGroupReplyToBotTriggersScenario: WhatsAppQaScenarioImplementation = { + posture: "user-path", + configOverrides: { + groupPolicy: "open", + }, + requiresGroupJid: true, + buildRun: () => { + const suffix = randomUUID().slice(0, 8).toUpperCase(); + const seedMarker = `WHATSAPP_QA_REPLY_TO_BOT_SEED_${suffix}`; + const triggerMarker = `WHATSAPP_QA_REPLY_TO_BOT_TRIGGER_${suffix}`; + return { + afterReply: async (reply, context) => { + assertWhatsAppMessageFromSutPhone(reply, context); + const quotedStartedAt = new Date(); + const quotedTrigger = await context.driver.sendText( + context.target, + `Quoted implicit reply trigger marker ${triggerMarker}`, + { + quotedMessageKey: buildWhatsAppQuotedMessageKeyFromObservedMessage(reply, { + remoteJid: context.target, + }), + }, + ); + if (!quotedTrigger.messageId) { + throw new Error("WhatsApp driver did not return a quoted trigger message id."); + } + const quotedTriggerMessageId = quotedTrigger.messageId; + const quotedReply = await waitForWhatsAppScenarioSutMessage(context, { + diagnosticChecks: [ + { + label: "containsTriggerMarker", + match: (message) => message.text.includes(triggerMarker), + }, + { + label: "quotesTrigger", + match: (message) => message.quoted?.messageId === quotedTriggerMessageId, + }, + ], + match: (message) => + message.text.includes(triggerMarker) && + message.quoted?.messageId === quotedTriggerMessageId, + observedAfter: quotedStartedAt, + targetKind: "group", + timeoutMs: 60_000, + }); + assertWhatsAppMessageFromSutPhone(quotedReply, context); + return "quoted reply to bot triggered a group response without an explicit mention"; + }, + configMode: "allowlist", + expectReply: true, + input: `openclawqa Mentioned group seed marker ${seedMarker}`, + matchText: seedMarker, + target: "group", + }; + }, +}; + +export const whatsappQaTopLevelReplyShapeScenario: WhatsAppQaScenarioImplementation = { + posture: "user-path", + configOverrides: { + replyToMode: "off", + }, + buildRun: () => { + const token = `WHATSAPP_QA_TOP_LEVEL_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + configMode: "allowlist", + expectReply: true, + input: `Reply with only this exact marker: ${token}`, + matchText: token, + target: "dm", + verify: (reply) => { + if (reply.quoted?.messageId) { + throw new Error( + `expected top-level WhatsApp reply without quote metadata, got quoted message ${reply.quoted.messageId}`, + ); + } + }, + }; + }, +}; + +export const whatsappQaReplyToMessageScenario: WhatsAppQaScenarioImplementation = { + posture: "user-path", + configOverrides: { + replyToMode: "all", + }, + buildRun: () => buildWhatsAppQuoteReplyRun("dm"), +}; + +export const whatsappQaGroupReplyToMessageScenario: WhatsAppQaScenarioImplementation = { + posture: "user-path", + configOverrides: { + replyToMode: "all", + }, + requiresGroupJid: true, + buildRun: () => buildWhatsAppQuoteReplyRun("group"), +}; + +export const whatsappQaReplyToModeBatchedScenario: WhatsAppQaScenarioImplementation = { + posture: "user-path", + configOverrides: { + inboundDebounceMs: 250, + replyToMode: "batched", + }, + buildRun: () => { + const suffix = randomUUID().slice(0, 8).toUpperCase(); + const firstToken = `WHATSAPP_QA_BATCHED_FIRST_${suffix}`; + const finalToken = `WHATSAPP_QA_BATCHED_FINAL_${suffix}`; + let secondMessageId: string | undefined; + return { + afterSend: async (context) => { + const second = await context.driver.sendText( + context.target, + `Second batched WhatsApp QA message. Reply with only this exact marker: ${finalToken} only if the previous queued message is visible in this same run context.`, + ); + secondMessageId = second.messageId; + return "second batched message sent before debounce flush"; + }, + configMode: "allowlist", + expectReply: true, + input: `First batched WhatsApp QA message ${firstToken}. Wait for the next message before replying.`, + matchText: finalToken, + target: "dm", + verify: (reply) => { + if (!secondMessageId) { + throw new Error("WhatsApp driver did not return a second batched message id."); + } + if (reply.quoted?.messageId !== secondMessageId) { + throw new Error( + `expected batched reply quote ${secondMessageId}, got ${reply.quoted?.messageId ?? ""}`, + ); + } + }, + }; + }, +}; diff --git a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenario-implementations.delivery.ts b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenario-implementations.delivery.ts new file mode 100644 index 000000000000..50d390d52a19 --- /dev/null +++ b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenario-implementations.delivery.ts @@ -0,0 +1,237 @@ +// QA Lab WhatsApp delivery-shape, status, and approval scenarios. +import { randomUUID } from "node:crypto"; +import type { WhatsAppQaScenarioImplementation } from "./whatsapp-live.contracts.js"; +import { + callWhatsAppGatewaySend, + waitForScenarioObservedMessage, + waitForWhatsAppSutReactionSequenceToTrigger, + waitForWhatsAppSutReactionToTrigger, +} from "./whatsapp-live.operations.js"; + +export const whatsappQaReplyDeliveryShapeScenario: WhatsAppQaScenarioImplementation = { + posture: "direct-gateway", + buildRun: () => { + const token = `WHATSAPP_QA_REPLY_SHAPE_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + afterReply: async (_reply, context) => { + if (!context.sent.messageId) { + throw new Error("WhatsApp driver did not return a triggering message id."); + } + const quotedTriggerMessageId = context.sent.messageId; + const chunkStartedAt = new Date(); + const longText = `${token}_LONG_BEGIN\n${"A".repeat(4_500)}\n${token}_LONG_END`; + await callWhatsAppGatewaySend(context, { + label: "long-reply", + message: longText, + replyToId: quotedTriggerMessageId, + }); + const firstChunk = await waitForScenarioObservedMessage(context, { + observedAfter: chunkStartedAt, + diagnosticChecks: [ + { + label: "longBeginMarker", + match: (message) => message.text.includes(`${token}_LONG_BEGIN`), + }, + { + label: "quotesTrigger", + match: (message) => message.quoted?.messageId === quotedTriggerMessageId, + }, + ], + match: (message) => + message.text.includes(`${token}_LONG_BEGIN`) && + message.quoted?.messageId === quotedTriggerMessageId, + }); + const secondChunk = await waitForScenarioObservedMessage(context, { + observedAfter: chunkStartedAt, + diagnosticChecks: [ + { + label: "longEndMarker", + match: (message) => message.text.includes(`${token}_LONG_END`), + }, + { + label: "quotesTrigger", + match: (message) => message.quoted?.messageId === quotedTriggerMessageId, + }, + ], + match: (message) => + message.messageId !== firstChunk.messageId && + message.text.includes(`${token}_LONG_END`) && + message.quoted?.messageId === quotedTriggerMessageId, + }); + return `long reply chunked across ${firstChunk.messageId ?? ""} and ${secondChunk.messageId ?? ""}`; + }, + configMode: "allowlist", + expectReply: true, + input: `Reply with only this exact marker before reply-shape checks: ${token}`, + matchText: token, + target: "dm", + }; + }, +}; + +export const whatsappQaStreamFinalMessageAccountingScenario: WhatsAppQaScenarioImplementation = { + posture: "user-path", + buildRun: () => ({ + configMode: "allowlist", + expectReply: true, + expectedJoinedSutTextIncludes: ["WHATSAPP-LONG-FINAL-BEGIN", "WHATSAPP-LONG-FINAL-END"], + expectedSutMessageCount: 2, + input: "WhatsApp long final QA check. Use the scripted long final response.", + matchText: "WHATSAPP-LONG-FINAL-BEGIN", + settleMs: 4_000, + target: "dm", + }), +}; + +export const whatsappQaApprovalExecDenyNativeScenario: WhatsAppQaScenarioImplementation = { + posture: "native-approval", + configOverrides: { + approvals: { + exec: true, + }, + }, + buildRun: () => ({ + approvalKind: "exec", + decision: "deny", + kind: "approval", + token: `WHATSAPP_QA_EXEC_DENY_${randomUUID().slice(0, 8).toUpperCase()}`, + }), +}; + +export const whatsappQaStatusReactionsScenario: WhatsAppQaScenarioImplementation = { + posture: "user-path", + configOverrides: { + statusReactions: true, + }, + buildRun: () => { + const token = `WHATSAPP_QA_STATUS_REACTION_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + afterSend: async (context) => { + const reaction = await waitForWhatsAppSutReactionToTrigger(context, { + expectation: { anyEmoji: true }, + timeoutMs: 30_000, + }); + return `status reaction ${reaction.reaction?.emoji ?? ""} observed`; + }, + configMode: "allowlist", + expectReply: true, + input: `Reply with only this exact marker after normal processing: ${token}`, + matchText: token, + target: "dm", + }; + }, +}; + +export const whatsappQaStatusReactionLifecycleScenario: WhatsAppQaScenarioImplementation = { + posture: "user-path", + configOverrides: { + statusReactions: true, + }, + buildRun: () => { + const token = `WHATSAPP_QA_STATUS_LIFECYCLE_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + afterReply: async (_reply, context) => { + const reactions = await waitForWhatsAppSutReactionSequenceToTrigger(context, { + emojis: ["👀", "✅"], + observedAfter: context.requestStartedAt, + timeoutMs: 60_000, + }); + for (const reaction of reactions) { + context.recordObservedMessage(reaction); + } + return `status reaction lifecycle observed ${reactions + .map((reaction) => reaction.reaction?.emoji ?? "") + .join(" -> ")}`; + }, + configMode: "allowlist", + expectReply: true, + input: `Reply with only this exact marker after normal processing: ${token}`, + matchText: token, + target: "dm", + }; + }, +}; + +export const whatsappQaGroupAllowlistBlockScenario: WhatsAppQaScenarioImplementation = { + posture: "user-path", + configOverrides: { + blockGroupSender: true, + groupPolicy: "allowlist", + }, + requiresGroupJid: true, + buildRun: () => { + const quietToken = `WHATSAPP_QA_GROUP_BLOCK_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + configMode: "allowlist", + expectReply: false, + input: `openclawqa blocked group should not reply with ${quietToken}`, + matchText: quietToken, + target: "group", + }; + }, +}; + +export const whatsappQaApprovalExecNativeScenario: WhatsAppQaScenarioImplementation = { + posture: "native-approval", + configOverrides: { + approvals: { + exec: true, + }, + }, + buildRun: () => ({ + approvalKind: "exec", + decision: "allow-once", + kind: "approval", + token: `WHATSAPP_QA_EXEC_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`, + }), +}; + +export const whatsappQaApprovalExecReactionNativeScenario: WhatsAppQaScenarioImplementation = { + posture: "native-approval", + configOverrides: { + approvals: { + exec: true, + }, + }, + buildRun: () => ({ + approvalKind: "exec", + decision: "allow-once", + decisionMode: "reaction", + kind: "approval", + token: `WHATSAPP_QA_EXEC_REACTION_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`, + }), +}; + +export const whatsappQaApprovalExecGroupReactionNativeScenario: WhatsAppQaScenarioImplementation = { + posture: "native-approval", + configOverrides: { + approvals: { + exec: true, + }, + }, + requiresGroupJid: true, + buildRun: () => ({ + approvalKind: "exec", + decision: "allow-once", + decisionMode: "reaction", + kind: "approval", + target: "group", + token: `WHATSAPP_QA_GROUP_EXEC_REACTION_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`, + }), +}; + +export const whatsappQaApprovalPluginNativeScenario: WhatsAppQaScenarioImplementation = { + posture: "native-approval", + configOverrides: { + approvals: { + exec: true, + plugin: true, + }, + }, + buildRun: () => ({ + approvalKind: "plugin", + decision: "allow-once", + kind: "approval", + token: `WHATSAPP_QA_PLUGIN_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`, + }), +}; diff --git a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenario-implementations.user-path.ts b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenario-implementations.user-path.ts new file mode 100644 index 000000000000..065bd6b9af6d --- /dev/null +++ b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenario-implementations.user-path.ts @@ -0,0 +1,382 @@ +// QA Lab WhatsApp user-path action and inbound media scenarios. +import { randomUUID } from "node:crypto"; +import type { WhatsAppQaScenarioImplementation } from "./whatsapp-live.contracts.js"; +import { + WHATSAPP_QA_AUDIO_OGG_OPUS_MIME, + WHATSAPP_QA_AUDIO_TRANSCRIPT_MARKER, + WHATSAPP_QA_ONE_PIXEL_PNG, + assertWhatsAppMessageFromSutPhone, + callWhatsAppGatewaySend, + createWhatsAppQaAudioOggOpusBuffer, + createWhatsAppQaAudioWavBuffer, + createWhatsAppQaPdfBuffer, + matchesWhatsAppSutReactionToTrigger, + waitForNoWhatsAppReply, + waitForScenarioObservedMessage, + waitForWhatsAppScenarioSutMessage, + waitForWhatsAppSutReactionToTrigger, + writeWhatsAppQaWorkspaceFixture, +} from "./whatsapp-live.operations.js"; + +export const whatsappQaAgentMessageActionReactScenario: WhatsAppQaScenarioImplementation = { + posture: "user-path", + configOverrides: { + actions: true, + }, + buildRun: () => { + const token = `WHATSAPP_QA_AGENT_REACT_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + afterSend: async (context) => { + const reaction = await waitForWhatsAppSutReactionToTrigger(context, { + expectation: { emoji: "👍" }, + timeoutMs: 60_000, + }); + return `agent message reaction ${reaction.reaction?.emoji ?? ""} observed`; + }, + allowQuietWindowMessage: (message, context) => + matchesWhatsAppSutReactionToTrigger(message, context, { emoji: "👍" }), + configMode: "allowlist", + expectReply: false, + input: + `React to this WhatsApp message with thumbs up for QA action check ${token}. ` + + "Do not send any visible text reply after the reaction.", + matchText: token, + quietWindowMs: 8_000, + target: "dm", + }; + }, +}; + +export const whatsappQaAgentMessageActionUploadFileScenario: WhatsAppQaScenarioImplementation = { + posture: "user-path", + configOverrides: { + actions: true, + }, + buildRun: () => { + const token = `WHATSAPP_QA_AGENT_UPLOAD_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + afterSend: async (context) => { + const media = await waitForScenarioObservedMessage(context, { + observedAfter: context.requestStartedAt, + timeoutMs: 60_000, + match: (message) => + message.kind === "media" && + message.hasMedia === true && + message.mediaType?.startsWith("image/") === true && + message.text.includes(token), + }); + return `agent upload-file media ${media.mediaType ?? ""} observed`; + }, + allowQuietWindowMessage: (message) => + message.kind === "media" && + message.mediaType?.startsWith("image/") === true && + message.text.includes(token), + configMode: "allowlist", + expectReply: false, + input: + `Use the WhatsApp message tool upload-file action to send a PNG with caption ${token}. ` + + "Do not send any visible text reply after the upload.", + matchText: token, + quietWindowMs: 8_000, + target: "dm", + }; + }, +}; + +export const whatsappQaGroupAgentMessageActionReactScenario: WhatsAppQaScenarioImplementation = { + posture: "user-path", + configOverrides: { + actions: true, + }, + requiresGroupJid: true, + buildRun: () => { + const token = `WHATSAPP_QA_GROUP_AGENT_REACT_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + afterSend: async (context) => { + const reaction = await waitForWhatsAppSutReactionToTrigger(context, { + expectation: { emoji: "👍" }, + timeoutMs: 60_000, + }); + return `group agent message reaction ${reaction.reaction?.emoji ?? ""} observed`; + }, + allowQuietWindowMessage: (message, context) => + matchesWhatsAppSutReactionToTrigger(message, context, { emoji: "👍" }), + configMode: "allowlist", + expectReply: false, + input: + `openclawqa react to this WhatsApp group message with thumbs up for QA action check ${token}. ` + + "Do not send any visible text reply after the reaction.", + matchText: token, + quietWindowMs: 8_000, + target: "group", + }; + }, +}; + +export const whatsappQaGroupAgentMessageActionUploadFileScenario: WhatsAppQaScenarioImplementation = + { + posture: "user-path", + configOverrides: { + actions: true, + }, + requiresGroupJid: true, + buildRun: () => { + const token = `WHATSAPP_QA_GROUP_AGENT_UPLOAD_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + afterSend: async (context) => { + const media = await waitForWhatsAppScenarioSutMessage(context, { + observedAfter: context.requestStartedAt, + targetKind: "group", + timeoutMs: 60_000, + match: (message) => + message.kind === "media" && + message.hasMedia === true && + message.mediaType?.startsWith("image/") === true && + message.text.includes(token), + }); + return `group agent upload-file media ${media.mediaType ?? ""} observed`; + }, + allowQuietWindowMessage: (message) => + message.kind === "media" && + message.mediaType?.startsWith("image/") === true && + message.text.includes(token), + configMode: "allowlist", + expectReply: false, + input: + `openclawqa use the WhatsApp message tool upload-file action to send a PNG with caption ${token}. ` + + "Do not send any visible text reply after the upload.", + matchText: token, + quietWindowMs: 8_000, + target: "group", + }; + }, + }; + +export const whatsappQaInboundReactionNoTriggerScenario: WhatsAppQaScenarioImplementation = { + posture: "user-path", + buildRun: () => { + const token = `WHATSAPP_QA_INBOUND_REACTION_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + afterReply: async (reply, context) => { + assertWhatsAppMessageFromSutPhone(reply, context); + if (!reply.messageId) { + throw new Error("WhatsApp SUT reply did not include a message id to react to."); + } + const reactionStartedAt = new Date(); + await context.driver.sendReaction(context.target, reply.messageId, "❤️", { + fromMe: false, + }); + await waitForNoWhatsAppReply({ + driver: context.driver, + observedAfter: reactionStartedAt, + sutPhoneE164: context.sutPhoneE164, + target: "dm", + windowMs: 5_000, + }); + return "driver reaction to SUT message did not trigger a fresh reply"; + }, + configMode: "allowlist", + expectReply: true, + input: `Reply with only this exact marker before inbound reaction check: ${token}`, + matchText: token, + target: "dm", + }; + }, +}; + +export const whatsappQaReplyContextIsolationScenario: WhatsAppQaScenarioImplementation = { + posture: "direct-gateway", + buildRun: () => { + const token = `WHATSAPP_QA_REPLY_ISOLATION_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + afterReply: async (_reply, context) => { + if (!context.sent.messageId) { + throw new Error("WhatsApp driver did not return a triggering message id."); + } + const quotedStartedAt = new Date(); + await callWhatsAppGatewaySend(context, { + label: "quoted", + message: `${token}_QUOTED`, + replyToId: context.sent.messageId, + }); + await waitForScenarioObservedMessage(context, { + observedAfter: quotedStartedAt, + diagnosticChecks: [ + { + label: "textMarker", + match: (message) => message.text.includes(`${token}_QUOTED`), + }, + { + label: "quotedMessageIdMatchesTrigger", + match: (message) => message.quoted?.messageId === context.sent.messageId, + }, + ], + match: (message) => + message.text.includes(`${token}_QUOTED`) && + message.quoted?.messageId === context.sent.messageId, + }); + + const freshStartedAt = new Date(); + await callWhatsAppGatewaySend(context, { + label: "fresh", + message: `${token}_FRESH`, + }); + const fresh = await waitForScenarioObservedMessage(context, { + observedAfter: freshStartedAt, + match: (message) => message.text.includes(`${token}_FRESH`), + }); + if (fresh.quoted?.messageId) { + throw new Error( + `expected fresh WhatsApp send without quote metadata, got quoted message ${fresh.quoted.messageId}`, + ); + } + return "quoted send and fresh send used independent reply context"; + }, + configMode: "allowlist", + expectReply: true, + input: `Reply with only this exact marker before reply isolation checks: ${token}`, + matchText: token, + target: "dm", + }; + }, +}; + +export const whatsappQaInboundImageCaptionScenario: WhatsAppQaScenarioImplementation = { + posture: "user-path", + buildRun: () => { + const token = `WHATSAPP_QA_IMAGE_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + configMode: "allowlist", + expectReply: true, + input: `This image caption asks you to reply with only this exact marker: ${token}`, + matchText: token, + sendMode: { + fileName: "whatsapp-qa.png", + kind: "media", + mediaBuffer: WHATSAPP_QA_ONE_PIXEL_PNG, + mediaType: "image/png", + }, + target: "dm", + }; + }, +}; + +export const whatsappQaAudioPreflightScenario: WhatsAppQaScenarioImplementation = { + posture: "user-path", + configOverrides: { + audioPreflight: true, + }, + buildRun: () => ({ + configMode: "allowlist", + expectReply: true, + input: "", + matchText: WHATSAPP_QA_AUDIO_TRANSCRIPT_MARKER, + sendMode: { + fileName: "whatsapp-qa-audio.ogg", + kind: "media", + mediaBuffer: createWhatsAppQaAudioOggOpusBuffer(), + mediaType: WHATSAPP_QA_AUDIO_OGG_OPUS_MIME, + }, + target: "dm", + }), +}; + +export const whatsappQaOutboundMediaMatrixScenario: WhatsAppQaScenarioImplementation = { + posture: "direct-gateway", + buildRun: () => { + const token = `WHATSAPP_QA_OUTBOUND_MEDIA_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + afterReply: async (_reply, context) => { + const mediaRootToken = randomUUID().slice(0, 8); + const imagePath = await writeWhatsAppQaWorkspaceFixture(context, { + buffer: WHATSAPP_QA_ONE_PIXEL_PNG, + fileName: `whatsapp-qa-${mediaRootToken}.png`, + }); + const documentPath = await writeWhatsAppQaWorkspaceFixture(context, { + buffer: createWhatsAppQaPdfBuffer(), + fileName: `whatsapp-qa-${mediaRootToken}.pdf`, + }); + const audioPath = await writeWhatsAppQaWorkspaceFixture(context, { + buffer: createWhatsAppQaAudioWavBuffer(), + fileName: `whatsapp-qa-${mediaRootToken}.wav`, + }); + + const imageStartedAt = new Date(); + await callWhatsAppGatewaySend(context, { + label: "image", + mediaUrl: imagePath, + message: `${token}_IMAGE`, + }); + await waitForScenarioObservedMessage(context, { + observedAfter: imageStartedAt, + match: (message) => + message.kind === "media" && + message.hasMedia === true && + message.mediaType?.startsWith("image/") === true && + message.text.includes(`${token}_IMAGE`), + }); + + const documentStartedAt = new Date(); + await callWhatsAppGatewaySend(context, { + forceDocument: true, + label: "document", + mediaUrl: documentPath, + message: `${token}_DOCUMENT`, + }); + await waitForScenarioObservedMessage(context, { + observedAfter: documentStartedAt, + match: (message) => + message.kind === "media" && + message.hasMedia === true && + (message.mediaType === "application/pdf" || + message.mediaFileName?.endsWith(".pdf") === true) && + message.text.includes(`${token}_DOCUMENT`), + }); + + const audioStartedAt = new Date(); + await callWhatsAppGatewaySend(context, { + asVoice: true, + label: "audio", + mediaUrl: audioPath, + message: `${token}_AUDIO`, + }); + await waitForScenarioObservedMessage(context, { + observedAfter: audioStartedAt, + match: (message) => + message.kind === "media" && + message.hasMedia === true && + message.mediaType?.startsWith("audio/") === true, + }); + await waitForScenarioObservedMessage(context, { + observedAfter: audioStartedAt, + match: (message) => message.text.includes(`${token}_AUDIO`), + }); + + const multiStartedAt = new Date(); + await callWhatsAppGatewaySend(context, { + label: "multi", + mediaUrls: [imagePath, documentPath], + message: `${token}_MULTI`, + }); + await waitForScenarioObservedMessage(context, { + observedAfter: multiStartedAt, + match: (message) => + message.kind === "media" && message.mediaType?.startsWith("image/") === true, + }); + await waitForScenarioObservedMessage(context, { + observedAfter: multiStartedAt, + match: (message) => + message.kind === "media" && + (message.mediaType === "application/pdf" || + message.mediaFileName?.endsWith(".pdf") === true), + }); + return "gateway send delivered image, document, audio, and multi-media"; + }, + configMode: "allowlist", + expectReply: true, + input: `Reply with only this exact marker before outbound media checks: ${token}`, + matchText: token, + target: "dm", + }; + }, +}; diff --git a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenarios.capabilities.ts b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenarios.capabilities.ts deleted file mode 100644 index 5bd05ef84809..000000000000 --- a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenarios.capabilities.ts +++ /dev/null @@ -1,420 +0,0 @@ -// QA Lab WhatsApp Gateway capability and structured-message scenarios. -import { randomUUID } from "node:crypto"; -import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; -import type { WhatsAppQaScenarioDefinition } from "./whatsapp-live.contracts.js"; -import { - WHATSAPP_QA_AUDIO_OGG_OPUS_MIME, - WHATSAPP_QA_GROUP_AUDIO_TRANSCRIPT_MARKER, - WHATSAPP_QA_ONE_PIXEL_PNG, - callWhatsAppGatewayMessageAction, - callWhatsAppGatewayPoll, - callWhatsAppGatewaySend, - callWhatsAppGatewaySendConcurrently, - createWhatsAppQaAudioOggOpusBuffer, - createWhatsAppQaPdfBuffer, - requireWhatsAppTriggerMessageId, - runWhatsAppStructuredInboundChecks, - waitForScenarioObservedMessage, - waitForWhatsAppScenarioSutMessage, - waitForWhatsAppSutReactionToTrigger, - writeWhatsAppQaWorkspaceFixture, -} from "./whatsapp-live.operations.js"; - -export const WHATSAPP_QA_CAPABILITY_SCENARIOS: WhatsAppQaScenarioDefinition[] = [ - { - id: "whatsapp-outbound-document-preserves-filename", - title: "WhatsApp direct Gateway document preserves filename and caption", - timeoutMs: 90_000, - buildRun: () => { - const token = `WHATSAPP_QA_DOCUMENT_FILE_${randomUUID().slice(0, 8).toUpperCase()}`; - return { - afterReply: async (_reply, context) => { - const documentPath = await writeWhatsAppQaWorkspaceFixture(context, { - buffer: createWhatsAppQaPdfBuffer(), - fileName: `whatsapp-qa-report-${token}.pdf`, - }); - const documentStartedAt = new Date(); - await callWhatsAppGatewaySend(context, { - forceDocument: true, - label: "document-filename", - mediaUrl: documentPath, - message: `${token}_CAPTION`, - }); - const document = await waitForScenarioObservedMessage(context, { - observedAfter: documentStartedAt, - match: (message) => - message.kind === "media" && - message.hasMedia === true && - message.text.includes(`${token}_CAPTION`) && - message.mediaFileName === `whatsapp-qa-report-${token}.pdf`, - }); - return `document ${document.mediaFileName ?? ""} preserved`; - }, - configMode: "allowlist", - expectReply: true, - input: `Reply with only this exact marker before document filename check: ${token}`, - matchText: token, - target: "dm", - }; - }, - }, - { - id: "whatsapp-outbound-send-serialization", - title: "WhatsApp parallel Gateway sends deliver every outbound message", - timeoutMs: 90_000, - buildRun: () => { - const token = `WHATSAPP_QA_SERIAL_SEND_${randomUUID().slice(0, 8).toUpperCase()}`; - const markers = Array.from({ length: 5 }, (_, index) => `${token}_${index + 1}`); - return { - afterReply: async (_reply, context) => { - const sendsStartedAt = new Date(); - await callWhatsAppGatewaySendConcurrently( - context, - markers.map((marker, index) => ({ - label: `parallel-${index + 1}`, - message: marker, - })), - ); - await Promise.all( - markers.map((marker) => - waitForScenarioObservedMessage(context, { - observedAfter: sendsStartedAt, - match: (message) => message.kind === "text" && message.text.includes(marker), - }), - ), - ); - return `gateway parallel send delivered ${markers.length}/${markers.length} messages`; - }, - configMode: "allowlist", - expectReply: true, - input: `Reply with only this exact marker before parallel send checks: ${token}`, - matchText: token, - target: "dm", - }; - }, - }, - { - id: "whatsapp-outbound-poll", - title: "WhatsApp direct Gateway poll delivers outbound native poll", - timeoutMs: 90_000, - buildRun: () => { - const token = `WHATSAPP_QA_OUTBOUND_POLL_${randomUUID().slice(0, 8).toUpperCase()}`; - const question = `${token} choose one`; - return { - afterReply: async (_reply, context) => { - const pollStartedAt = new Date(); - await callWhatsAppGatewayPoll(context, { - label: "poll", - options: ["alpha", "beta"], - question, - }); - const poll = await waitForScenarioObservedMessage(context, { - observedAfter: pollStartedAt, - match: (message) => - message.kind === "poll" && - message.poll?.question === question && - message.poll.options.includes("alpha") && - message.poll.options.includes("beta"), - }); - return `poll observed with ${poll.poll?.options.length ?? 0} options`; - }, - configMode: "allowlist", - expectReply: true, - input: `Reply with only this exact marker before outbound poll check: ${token}`, - matchText: token, - target: "dm", - }; - }, - }, - { - id: "whatsapp-group-outbound-media", - title: "WhatsApp direct Gateway send delivers media to a group", - timeoutMs: 120_000, - requiresGroupJid: true, - buildRun: () => { - const token = `WHATSAPP_QA_GROUP_OUTBOUND_MEDIA_${randomUUID().slice(0, 8).toUpperCase()}`; - return { - afterReply: async (_reply, context) => { - const mediaRootToken = randomUUID().slice(0, 8); - const imagePath = await writeWhatsAppQaWorkspaceFixture(context, { - buffer: WHATSAPP_QA_ONE_PIXEL_PNG, - fileName: `whatsapp-qa-group-${mediaRootToken}.png`, - }); - const documentPath = await writeWhatsAppQaWorkspaceFixture(context, { - buffer: createWhatsAppQaPdfBuffer(), - fileName: `whatsapp-qa-group-${mediaRootToken}.pdf`, - }); - - const imageStartedAt = new Date(); - await callWhatsAppGatewaySend(context, { - label: "group-image", - mediaUrl: imagePath, - message: `${token}_IMAGE`, - }); - await waitForWhatsAppScenarioSutMessage(context, { - observedAfter: imageStartedAt, - targetKind: "group", - match: (message) => - message.kind === "media" && - message.hasMedia === true && - message.mediaType?.startsWith("image/") === true && - message.text.includes(`${token}_IMAGE`), - }); - - const documentStartedAt = new Date(); - await callWhatsAppGatewaySend(context, { - forceDocument: true, - label: "group-document", - mediaUrl: documentPath, - message: `${token}_DOCUMENT`, - }); - await waitForWhatsAppScenarioSutMessage(context, { - observedAfter: documentStartedAt, - targetKind: "group", - match: (message) => - message.kind === "media" && - message.hasMedia === true && - (message.mediaType === "application/pdf" || - message.mediaFileName?.endsWith(".pdf") === true) && - message.text.includes(`${token}_DOCUMENT`), - }); - return "gateway send delivered image and document media to the group"; - }, - configMode: "allowlist", - expectReply: true, - input: `openclawqa reply with only this exact marker before group outbound media checks: ${token}`, - matchText: token, - target: "group", - }; - }, - }, - { - id: "whatsapp-group-outbound-audio", - title: "WhatsApp direct Gateway send delivers audio to a group", - timeoutMs: 90_000, - requiresGroupJid: true, - buildRun: () => { - const token = `WHATSAPP_QA_GROUP_OUTBOUND_AUDIO_${randomUUID().slice(0, 8).toUpperCase()}`; - return { - afterReply: async (_reply, context) => { - const audioPath = await writeWhatsAppQaWorkspaceFixture(context, { - buffer: createWhatsAppQaAudioOggOpusBuffer({ variant: "group-trigger" }), - fileName: `whatsapp-qa-group-audio-${token}.ogg`, - }); - const audioStartedAt = new Date(); - await callWhatsAppGatewaySend(context, { - asVoice: true, - label: "group-audio", - mediaUrl: audioPath, - message: `${token}_AUDIO`, - }); - await waitForWhatsAppScenarioSutMessage(context, { - observedAfter: audioStartedAt, - targetKind: "group", - match: (message) => - message.kind === "media" && - message.hasMedia === true && - message.mediaType?.startsWith("audio/") === true, - }); - await waitForWhatsAppScenarioSutMessage(context, { - observedAfter: audioStartedAt, - targetKind: "group", - match: (message) => message.text.includes(`${token}_AUDIO`), - }); - return "gateway send delivered audio media to the group"; - }, - configMode: "allowlist", - expectReply: true, - input: `openclawqa reply with only this exact marker before group outbound audio check: ${token}`, - matchText: token, - target: "group", - }; - }, - }, - { - id: "whatsapp-group-outbound-poll", - title: "WhatsApp direct Gateway poll delivers native poll to a group", - timeoutMs: 90_000, - requiresGroupJid: true, - buildRun: () => { - const token = `WHATSAPP_QA_GROUP_OUTBOUND_POLL_${randomUUID().slice(0, 8).toUpperCase()}`; - const question = `${token} choose one`; - return { - afterReply: async (_reply, context) => { - const pollStartedAt = new Date(); - await callWhatsAppGatewayPoll(context, { - label: "group-poll", - options: ["alpha", "beta"], - question, - }); - const poll = await waitForWhatsAppScenarioSutMessage(context, { - observedAfter: pollStartedAt, - targetKind: "group", - match: (message) => - message.kind === "poll" && - message.poll?.question === question && - message.poll.options.includes("alpha") && - message.poll.options.includes("beta"), - }); - return `group poll observed with ${poll.poll?.options.length ?? 0} options`; - }, - configMode: "allowlist", - expectReply: true, - input: `openclawqa reply with only this exact marker before group outbound poll check: ${token}`, - matchText: token, - target: "group", - }; - }, - }, - { - id: "whatsapp-message-actions", - title: "WhatsApp direct Gateway message.action react and upload-file execute", - timeoutMs: 120_000, - configOverrides: { - actions: true, - }, - buildRun: () => { - const token = `WHATSAPP_QA_ACTIONS_${randomUUID().slice(0, 8).toUpperCase()}`; - return { - afterReply: async (_reply, context) => { - const triggerMessageId = requireWhatsAppTriggerMessageId(context); - const reactionStartedAt = new Date(); - await callWhatsAppGatewayMessageAction(context, { - action: "react", - label: "react", - params: { - emoji: "👍", - messageId: triggerMessageId, - }, - }); - await waitForWhatsAppSutReactionToTrigger(context, { - expectation: { emoji: "👍" }, - observedAfter: reactionStartedAt, - }); - - const uploadStartedAt = new Date(); - await callWhatsAppGatewayMessageAction(context, { - action: "upload-file", - label: "upload-file", - params: { - buffer: WHATSAPP_QA_ONE_PIXEL_PNG.toString("base64"), - caption: `${token}_UPLOAD`, - contentType: "image/png", - filename: "whatsapp-qa-upload.png", - }, - }); - await waitForScenarioObservedMessage(context, { - observedAfter: uploadStartedAt, - match: (message) => - message.kind === "media" && - message.mediaType?.startsWith("image/") === true && - message.text.includes(`${token}_UPLOAD`), - }); - return "message.action react and upload-file observed"; - }, - configMode: "allowlist", - expectReply: true, - input: `Reply with only this exact marker before action checks: ${token}`, - matchText: token, - target: "dm", - }; - }, - }, - { - id: "whatsapp-inbound-structured-messages", - title: "WhatsApp inbound structured messages reach the agent", - timeoutMs: 240_000, - buildRun: () => { - const token = `WHATSAPP_QA_STRUCTURED_${randomUUID().slice(0, 8).toUpperCase()}`; - const locationToken = `${token}_LOCATION`; - const contactToken = `${token}_CONTACT`; - const stickerToken = `${token}_STICKER`; - const locationCoordinateText = "37.774900, -122.419400"; - return { - afterReply: async (_reply, context) => { - const waitForStructuredReply = async ( - label: string, - observedAfter: Date, - expectedToken: string, - ) => { - try { - return await waitForScenarioObservedMessage(context, { - observedAfter, - timeoutMs: 60_000, - match: (message) => message.text.includes(expectedToken), - diagnosticChecks: [ - { - label: "containsExpectedToken", - match: (message) => message.text.includes(expectedToken), - }, - ], - }); - } catch (error) { - throw new Error( - `timed out waiting for WhatsApp structured ${label} reply (${expectedToken}): ${formatErrorMessage(error)}`, - { cause: error }, - ); - } - }; - - await runWhatsAppStructuredInboundChecks({ - contactToken, - documentToken: `${token}_DOCUMENT`, - driver: context.driver, - driverPhoneE164: context.driverPhoneE164, - locationToken, - stickerToken, - target: context.target, - waitForStructuredReply, - }); - return "document, location, contact, and sticker elicited replies"; - }, - configMode: "allowlist", - expectReply: true, - input: - `When a later WhatsApp location message shows ${locationCoordinateText}, ` + - `reply with only this WhatsApp location marker: ${locationToken}. ` + - `When a later WhatsApp contact message appears, ` + - `reply with only this WhatsApp contact marker: ${contactToken}. ` + - `When a later WhatsApp sticker message appears, ` + - `reply with only this WhatsApp sticker marker: ${stickerToken}. ` + - `Reply with only this exact marker before structured inbound checks: ${token}`, - matchText: token, - target: "dm", - }; - }, - }, - { - id: "whatsapp-group-audio-gating", - title: "WhatsApp group audio mention gating", - timeoutMs: 120_000, - configOverrides: { - audioPreflight: true, - }, - requiredPluginIds: ["openai"], - requiresGroupJid: true, - buildRun: () => ({ - configMode: "allowlist", - expectReply: true, - input: "", - matchText: WHATSAPP_QA_GROUP_AUDIO_TRANSCRIPT_MARKER, - quietInput: "", - quietSendMode: { - fileName: "whatsapp-qa-group-audio-quiet.ogg", - kind: "media", - mediaBuffer: createWhatsAppQaAudioOggOpusBuffer(), - mediaType: WHATSAPP_QA_AUDIO_OGG_OPUS_MIME, - }, - quietWindowMs: 5_000, - sendMode: { - fileName: "whatsapp-qa-group-audio.ogg", - kind: "media", - mediaBuffer: createWhatsAppQaAudioOggOpusBuffer({ - variant: "group-trigger", - }), - mediaType: WHATSAPP_QA_AUDIO_OGG_OPUS_MIME, - }, - target: "group", - }), - }, -]; diff --git a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenarios.conversation.ts b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenarios.conversation.ts deleted file mode 100644 index d4c98794ebf6..000000000000 --- a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenarios.conversation.ts +++ /dev/null @@ -1,379 +0,0 @@ -// QA Lab WhatsApp conversation and reply-context scenarios. -import { randomUUID } from "node:crypto"; -import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; -import { - toWhatsAppQaError, - type WhatsAppQaMessageScenarioRun, - type WhatsAppQaScenarioDefinition, -} from "./whatsapp-live.contracts.js"; -import { - assertWhatsAppMessageFromSutPhone, - assertWhatsAppMessagesFromSutPhone, - buildWhatsAppQuotedMessageKeyFromObservedMessage, - resolveWhatsAppQaNoReplyTarget, - waitForDistinctWhatsAppSutMessages, - waitForNoWhatsAppReply, - waitForWhatsAppScenarioSutMessage, -} from "./whatsapp-live.operations.js"; - -function buildWhatsAppQuoteReplyRun(target: "dm" | "group"): WhatsAppQaMessageScenarioRun { - const token = `WHATSAPP_QA_REPLY_TO_${target.toUpperCase()}_${randomUUID().slice(0, 8).toUpperCase()}`; - const input = - target === "group" - ? `openclawqa reply with only this exact marker: ${token}` - : `Reply with only this exact marker: ${token}`; - return { - configMode: "allowlist", - expectReply: true, - input, - matchText: token, - target, - verify: (reply, context) => { - if (!context.sent.messageId) { - throw new Error("WhatsApp driver did not return a triggering message id."); - } - if (reply.quoted?.messageId !== context.sent.messageId) { - throw new Error( - `expected reply quote ${context.sent.messageId}, got ${reply.quoted?.messageId ?? ""}`, - ); - } - }, - }; -} - -export const WHATSAPP_QA_CONVERSATION_SCENARIOS: WhatsAppQaScenarioDefinition[] = [ - { - id: "whatsapp-canary", - title: "WhatsApp DM canary", - timeoutMs: 60_000, - buildRun: () => { - const token = `WHATSAPP_QA_ECHO_${randomUUID().slice(0, 8).toUpperCase()}`; - return { - configMode: "allowlist", - expectReply: true, - input: `Reply with only this exact marker: ${token}`, - matchText: token, - target: "dm", - }; - }, - }, - { - id: "whatsapp-mention-gating", - title: "WhatsApp group mention gating", - timeoutMs: 60_000, - requiresGroupJid: true, - buildRun: () => { - const quietToken = `WHATSAPP_QA_GROUP_QUIET_${randomUUID().slice(0, 8).toUpperCase()}`; - const replyToken = `WHATSAPP_QA_GROUP_MENTION_${randomUUID().slice(0, 8).toUpperCase()}`; - return { - configMode: "allowlist", - expectReply: true, - input: `openclawqa reply with only this exact marker: ${replyToken}`, - matchText: replyToken, - quietInput: `This group message is intentionally unmentioned. If you respond, include ${quietToken}.`, - quietMatchText: quietToken, - quietWindowMs: 5_000, - target: "group", - }; - }, - }, - { - id: "whatsapp-group-pending-history-context", - title: "WhatsApp group pending history reaches mentioned turns", - timeoutMs: 90_000, - configOverrides: { - groupHistoryLimit: 50, - groupPolicy: "open", - inboundDebounceMs: 0, - replyToMode: "all", - }, - requiresGroupJid: true, - buildRun: () => { - const suffix = randomUUID().slice(0, 8).toUpperCase(); - const quietMarker = `WHATSAPP_QA_PENDING_HISTORY_QUIET_${suffix}`; - const contextSentinel = `WHATSAPP_QA_PENDING_HISTORY_CONTEXT_ONLY_${suffix}`; - const triggerMarker = `WHATSAPP_QA_PENDING_HISTORY_TRIGGER_${suffix}`; - const okMarker = `WHATSAPP_QA_PENDING_HISTORY_OK_${suffix}`; - return { - configMode: "open", - expectReply: true, - expectedSutMessageCount: 1, - input: - `openclawqa pending history context check ${triggerMarker}. ` + - `Reply with only ${okMarker} only if the previous quiet group message is present ` + - `in prior group context with its context-only sentinel. ` + - "Do not use current-message text as proof.", - matchText: okMarker, - quietInput: `quiet context marker ${quietMarker} ${contextSentinel}`, - quietWindowMs: 5_000, - target: "group", - }; - }, - }, - { - id: "whatsapp-broadcast-group-fanout", - title: "WhatsApp group broadcast fans out to multiple agents", - timeoutMs: 120_000, - configOverrides: { - broadcast: { - agents: ["main", "qa-second"], - strategy: "sequential", - }, - groupPolicy: "open", - }, - requiresGroupJid: true, - buildRun: () => { - const token = `WHATSAPP_QA_BROADCAST_TOKEN_${randomUUID().slice(0, 8).toUpperCase()}`; - const mainMarker = `${token}_MAIN`; - const secondMarker = `${token}_SECOND`; - return { - afterReply: async (reply, context) => { - const replies = await waitForDistinctWhatsAppSutMessages(context, { - initialMessages: [reply], - matchers: [ - (message) => message.text.includes(mainMarker), - (message) => message.text.includes(secondMarker), - ], - observedAfter: context.requestStartedAt, - timeoutMs: 60_000, - }); - assertWhatsAppMessagesFromSutPhone(replies, context); - return "broadcast fanout produced main and qa-second replies"; - }, - configMode: "open", - expectReply: true, - input: `openclawqa broadcast fanout check ${token}`, - matchText: mainMarker, - target: "group", - }; - }, - }, - { - id: "whatsapp-group-activation-always", - title: "WhatsApp group activation always wakes unmentioned messages", - timeoutMs: 120_000, - configOverrides: { - groupPolicy: "open", - }, - requiresGroupJid: true, - buildRun: () => { - const suffix = randomUUID().slice(0, 8).toUpperCase(); - const alwaysMarker = `WHATSAPP_QA_ACTIVATION_ALWAYS_${suffix}`; - const quietMarker = `WHATSAPP_QA_ACTIVATION_QUIET_${suffix}`; - return { - afterReply: async (reply, context) => { - assertWhatsAppMessageFromSutPhone(reply, context); - let activationProbeError: unknown; - try { - const alwaysStartedAt = new Date(); - await context.driver.sendText( - context.target, - `Group activation visible behavior marker ${alwaysMarker}`, - ); - const alwaysReply = await waitForWhatsAppScenarioSutMessage(context, { - match: (message) => message.text.includes(alwaysMarker), - observedAfter: alwaysStartedAt, - targetKind: "group", - timeoutMs: 60_000, - }); - assertWhatsAppMessageFromSutPhone(alwaysReply, context); - } catch (error) { - activationProbeError = error; - } - - let restoreError: unknown; - const restoreStartedAt = new Date(); - try { - await context.driver.sendText(context.target, "/activation mention"); - const restoreReply = await waitForWhatsAppScenarioSutMessage(context, { - match: (message) => /\bactivation\b.*\bmention\b/iu.test(message.text), - observedAfter: restoreStartedAt, - targetKind: "group", - timeoutMs: 60_000, - }); - assertWhatsAppMessageFromSutPhone(restoreReply, context); - } catch (error) { - restoreError = error; - } - - if (activationProbeError && restoreError) { - throw new Error( - `activation always probe failed; additionally failed to restore mention mode: ${formatErrorMessage(restoreError)}`, - { cause: activationProbeError }, - ); - } - if (activationProbeError) { - throw toWhatsAppQaError(activationProbeError); - } - if (restoreError) { - throw toWhatsAppQaError(restoreError); - } - - const quietStartedAt = new Date(); - await context.driver.sendText( - context.target, - `Group activation quiet marker ${quietMarker}`, - ); - await waitForNoWhatsAppReply({ - driver: context.driver, - observedAfter: quietStartedAt, - sutPhoneE164: context.sutPhoneE164, - windowMs: 5_000, - ...resolveWhatsAppQaNoReplyTarget({ - groupJid: context.target, - target: "group", - }), - }); - return "activation always replied to an unmentioned group message and mention mode was restored"; - }, - configMode: "allowlist", - expectReply: true, - input: "/activation always", - matchText: /\bactivation\b.*\balways\b/iu, - target: "group", - }; - }, - }, - { - id: "whatsapp-group-reply-to-bot-triggers", - title: "WhatsApp group reply to bot wakes without an explicit mention", - timeoutMs: 120_000, - configOverrides: { - groupPolicy: "open", - }, - requiresGroupJid: true, - buildRun: () => { - const suffix = randomUUID().slice(0, 8).toUpperCase(); - const seedMarker = `WHATSAPP_QA_REPLY_TO_BOT_SEED_${suffix}`; - const triggerMarker = `WHATSAPP_QA_REPLY_TO_BOT_TRIGGER_${suffix}`; - return { - afterReply: async (reply, context) => { - assertWhatsAppMessageFromSutPhone(reply, context); - const quotedStartedAt = new Date(); - const quotedTrigger = await context.driver.sendText( - context.target, - `Quoted implicit reply trigger marker ${triggerMarker}`, - { - quotedMessageKey: buildWhatsAppQuotedMessageKeyFromObservedMessage(reply, { - remoteJid: context.target, - }), - }, - ); - if (!quotedTrigger.messageId) { - throw new Error("WhatsApp driver did not return a quoted trigger message id."); - } - const quotedTriggerMessageId = quotedTrigger.messageId; - const quotedReply = await waitForWhatsAppScenarioSutMessage(context, { - diagnosticChecks: [ - { - label: "containsTriggerMarker", - match: (message) => message.text.includes(triggerMarker), - }, - { - label: "quotesTrigger", - match: (message) => message.quoted?.messageId === quotedTriggerMessageId, - }, - ], - match: (message) => - message.text.includes(triggerMarker) && - message.quoted?.messageId === quotedTriggerMessageId, - observedAfter: quotedStartedAt, - targetKind: "group", - timeoutMs: 60_000, - }); - assertWhatsAppMessageFromSutPhone(quotedReply, context); - return "quoted reply to bot triggered a group response without an explicit mention"; - }, - configMode: "allowlist", - expectReply: true, - input: `openclawqa Mentioned group seed marker ${seedMarker}`, - matchText: seedMarker, - target: "group", - }; - }, - }, - { - id: "whatsapp-top-level-reply-shape", - title: "WhatsApp DM top-level reply shape", - timeoutMs: 60_000, - configOverrides: { - replyToMode: "off", - }, - buildRun: () => { - const token = `WHATSAPP_QA_TOP_LEVEL_${randomUUID().slice(0, 8).toUpperCase()}`; - return { - configMode: "allowlist", - expectReply: true, - input: `Reply with only this exact marker: ${token}`, - matchText: token, - target: "dm", - verify: (reply) => { - if (reply.quoted?.messageId) { - throw new Error( - `expected top-level WhatsApp reply without quote metadata, got quoted message ${reply.quoted.messageId}`, - ); - } - }, - }; - }, - }, - { - id: "whatsapp-reply-to-message", - title: "WhatsApp DM reply-to mode quotes the triggering message", - timeoutMs: 60_000, - configOverrides: { - replyToMode: "all", - }, - buildRun: () => buildWhatsAppQuoteReplyRun("dm"), - }, - { - id: "whatsapp-group-reply-to-message", - title: "WhatsApp group reply-to mode quotes the triggering message", - timeoutMs: 60_000, - configOverrides: { - replyToMode: "all", - }, - requiresGroupJid: true, - buildRun: () => buildWhatsAppQuoteReplyRun("group"), - }, - { - id: "whatsapp-reply-to-mode-batched", - title: "WhatsApp batched reply-to mode quotes the queued message", - timeoutMs: 90_000, - configOverrides: { - inboundDebounceMs: 250, - replyToMode: "batched", - }, - buildRun: () => { - const suffix = randomUUID().slice(0, 8).toUpperCase(); - const firstToken = `WHATSAPP_QA_BATCHED_FIRST_${suffix}`; - const finalToken = `WHATSAPP_QA_BATCHED_FINAL_${suffix}`; - let secondMessageId: string | undefined; - return { - afterSend: async (context) => { - const second = await context.driver.sendText( - context.target, - `Second batched WhatsApp QA message. Reply with only this exact marker: ${finalToken} only if the previous queued message is visible in this same run context.`, - ); - secondMessageId = second.messageId; - return "second batched message sent before debounce flush"; - }, - configMode: "allowlist", - expectReply: true, - input: `First batched WhatsApp QA message ${firstToken}. Wait for the next message before replying.`, - matchText: finalToken, - target: "dm", - verify: (reply) => { - if (!secondMessageId) { - throw new Error("WhatsApp driver did not return a second batched message id."); - } - if (reply.quoted?.messageId !== secondMessageId) { - throw new Error( - `expected batched reply quote ${secondMessageId}, got ${reply.quoted?.messageId ?? ""}`, - ); - } - }, - }; - }, - }, -]; diff --git a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenarios.delivery.ts b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenarios.delivery.ts deleted file mode 100644 index 3f6462fb62eb..000000000000 --- a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenarios.delivery.ts +++ /dev/null @@ -1,250 +0,0 @@ -// QA Lab WhatsApp delivery-shape, status, and approval scenarios. -import { randomUUID } from "node:crypto"; -import type { WhatsAppQaScenarioDefinition } from "./whatsapp-live.contracts.js"; -import { - callWhatsAppGatewaySend, - waitForScenarioObservedMessage, - waitForWhatsAppSutReactionSequenceToTrigger, - waitForWhatsAppSutReactionToTrigger, -} from "./whatsapp-live.operations.js"; - -export const WHATSAPP_QA_DELIVERY_SCENARIOS: WhatsAppQaScenarioDefinition[] = [ - { - id: "whatsapp-reply-delivery-shape", - title: "WhatsApp direct Gateway send chunks long replies", - timeoutMs: 120_000, - buildRun: () => { - const token = `WHATSAPP_QA_REPLY_SHAPE_${randomUUID().slice(0, 8).toUpperCase()}`; - return { - afterReply: async (_reply, context) => { - if (!context.sent.messageId) { - throw new Error("WhatsApp driver did not return a triggering message id."); - } - const quotedTriggerMessageId = context.sent.messageId; - const chunkStartedAt = new Date(); - const longText = `${token}_LONG_BEGIN\n${"A".repeat(4_500)}\n${token}_LONG_END`; - await callWhatsAppGatewaySend(context, { - label: "long-reply", - message: longText, - replyToId: quotedTriggerMessageId, - }); - const firstChunk = await waitForScenarioObservedMessage(context, { - observedAfter: chunkStartedAt, - diagnosticChecks: [ - { - label: "longBeginMarker", - match: (message) => message.text.includes(`${token}_LONG_BEGIN`), - }, - { - label: "quotesTrigger", - match: (message) => message.quoted?.messageId === quotedTriggerMessageId, - }, - ], - match: (message) => - message.text.includes(`${token}_LONG_BEGIN`) && - message.quoted?.messageId === quotedTriggerMessageId, - }); - const secondChunk = await waitForScenarioObservedMessage(context, { - observedAfter: chunkStartedAt, - diagnosticChecks: [ - { - label: "longEndMarker", - match: (message) => message.text.includes(`${token}_LONG_END`), - }, - { - label: "quotesTrigger", - match: (message) => message.quoted?.messageId === quotedTriggerMessageId, - }, - ], - match: (message) => - message.messageId !== firstChunk.messageId && - message.text.includes(`${token}_LONG_END`) && - message.quoted?.messageId === quotedTriggerMessageId, - }); - return `long reply chunked across ${firstChunk.messageId ?? ""} and ${secondChunk.messageId ?? ""}`; - }, - configMode: "allowlist", - expectReply: true, - input: `Reply with only this exact marker before reply-shape checks: ${token}`, - matchText: token, - target: "dm", - }; - }, - }, - { - id: "whatsapp-stream-final-message-accounting", - title: "WhatsApp streamed final response has exactly the final chunks", - timeoutMs: 90_000, - buildRun: () => ({ - configMode: "allowlist", - expectReply: true, - expectedJoinedSutTextIncludes: ["WHATSAPP-LONG-FINAL-BEGIN", "WHATSAPP-LONG-FINAL-END"], - expectedSutMessageCount: 2, - input: "WhatsApp long final QA check. Use the scripted long final response.", - matchText: "WHATSAPP-LONG-FINAL-BEGIN", - settleMs: 4_000, - target: "dm", - }), - }, - { - id: "whatsapp-approval-exec-deny-native", - title: "WhatsApp native exec approval prompt denies", - timeoutMs: 60_000, - configOverrides: { - approvals: { - exec: true, - }, - }, - buildRun: () => ({ - approvalKind: "exec", - decision: "deny", - kind: "approval", - token: `WHATSAPP_QA_EXEC_DENY_${randomUUID().slice(0, 8).toUpperCase()}`, - }), - }, - { - id: "whatsapp-status-reactions", - title: "WhatsApp status reactions are observable", - timeoutMs: 60_000, - configOverrides: { - statusReactions: true, - }, - buildRun: () => { - const token = `WHATSAPP_QA_STATUS_REACTION_${randomUUID().slice(0, 8).toUpperCase()}`; - return { - afterSend: async (context) => { - const reaction = await waitForWhatsAppSutReactionToTrigger(context, { - expectation: { anyEmoji: true }, - timeoutMs: 30_000, - }); - return `status reaction ${reaction.reaction?.emoji ?? ""} observed`; - }, - configMode: "allowlist", - expectReply: true, - input: `Reply with only this exact marker after normal processing: ${token}`, - matchText: token, - target: "dm", - }; - }, - }, - { - id: "whatsapp-status-reaction-lifecycle", - title: "WhatsApp status reaction lifecycle updates the triggering message", - timeoutMs: 90_000, - configOverrides: { - statusReactions: true, - }, - buildRun: () => { - const token = `WHATSAPP_QA_STATUS_LIFECYCLE_${randomUUID().slice(0, 8).toUpperCase()}`; - return { - afterReply: async (_reply, context) => { - const reactions = await waitForWhatsAppSutReactionSequenceToTrigger(context, { - emojis: ["👀", "✅"], - observedAfter: context.requestStartedAt, - timeoutMs: 60_000, - }); - for (const reaction of reactions) { - context.recordObservedMessage(reaction); - } - return `status reaction lifecycle observed ${reactions - .map((reaction) => reaction.reaction?.emoji ?? "") - .join(" -> ")}`; - }, - configMode: "allowlist", - expectReply: true, - input: `Reply with only this exact marker after normal processing: ${token}`, - matchText: token, - target: "dm", - }; - }, - }, - { - id: "whatsapp-group-allowlist-block", - title: "WhatsApp group outside allowlist stays quiet", - timeoutMs: 8_000, - configOverrides: { - blockGroupSender: true, - groupPolicy: "allowlist", - }, - requiresGroupJid: true, - buildRun: () => { - const quietToken = `WHATSAPP_QA_GROUP_BLOCK_${randomUUID().slice(0, 8).toUpperCase()}`; - return { - configMode: "allowlist", - expectReply: false, - input: `openclawqa blocked group should not reply with ${quietToken}`, - matchText: quietToken, - target: "group", - }; - }, - }, - { - id: "whatsapp-approval-exec-native", - title: "WhatsApp native exec approval prompt resolves", - timeoutMs: 60_000, - configOverrides: { - approvals: { - exec: true, - }, - }, - buildRun: () => ({ - approvalKind: "exec", - decision: "allow-once", - kind: "approval", - token: `WHATSAPP_QA_EXEC_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`, - }), - }, - { - id: "whatsapp-approval-exec-reaction-native", - title: "WhatsApp native exec approval resolves from reaction", - timeoutMs: 60_000, - configOverrides: { - approvals: { - exec: true, - }, - }, - buildRun: () => ({ - approvalKind: "exec", - decision: "allow-once", - decisionMode: "reaction", - kind: "approval", - token: `WHATSAPP_QA_EXEC_REACTION_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`, - }), - }, - { - id: "whatsapp-approval-exec-group-reaction-native", - title: "WhatsApp group-origin exec approval resolves from reaction", - timeoutMs: 60_000, - configOverrides: { - approvals: { - exec: true, - }, - }, - requiresGroupJid: true, - buildRun: () => ({ - approvalKind: "exec", - decision: "allow-once", - decisionMode: "reaction", - kind: "approval", - target: "group", - token: `WHATSAPP_QA_GROUP_EXEC_REACTION_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`, - }), - }, - { - id: "whatsapp-approval-plugin-native", - title: "WhatsApp native plugin approval prompt resolves with exec approvals enabled", - timeoutMs: 60_000, - configOverrides: { - approvals: { - exec: true, - plugin: true, - }, - }, - buildRun: () => ({ - approvalKind: "plugin", - decision: "allow-once", - kind: "approval", - token: `WHATSAPP_QA_PLUGIN_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`, - }), - }, -]; diff --git a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenarios.ts b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenarios.ts deleted file mode 100644 index 22de65ed9265..000000000000 --- a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenarios.ts +++ /dev/null @@ -1,21 +0,0 @@ -// QA Lab WhatsApp live scenario catalog and selection. -import type { WhatsAppQaScenarioDefinition } from "./whatsapp-live.contracts.js"; -import { WHATSAPP_QA_CAPABILITY_SCENARIOS } from "./whatsapp-live.scenarios.capabilities.js"; -import { WHATSAPP_QA_CONVERSATION_SCENARIOS } from "./whatsapp-live.scenarios.conversation.js"; -import { WHATSAPP_QA_DELIVERY_SCENARIOS } from "./whatsapp-live.scenarios.delivery.js"; -import { WHATSAPP_QA_USER_PATH_SCENARIOS } from "./whatsapp-live.scenarios.user-path.js"; - -const WHATSAPP_QA_SCENARIOS: WhatsAppQaScenarioDefinition[] = [ - ...WHATSAPP_QA_CONVERSATION_SCENARIOS, - ...WHATSAPP_QA_USER_PATH_SCENARIOS, - ...WHATSAPP_QA_CAPABILITY_SCENARIOS, - ...WHATSAPP_QA_DELIVERY_SCENARIOS, -]; - -export function getWhatsAppQaScenarioDefinition(id: string) { - const scenario = WHATSAPP_QA_SCENARIOS.find((candidate) => candidate.id === id); - if (!scenario) { - throw new Error(`unknown WhatsApp QA scenario id: ${id}`); - } - return scenario; -} diff --git a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenarios.user-path.ts b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenarios.user-path.ts deleted file mode 100644 index 6785865fb558..000000000000 --- a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenarios.user-path.ts +++ /dev/null @@ -1,394 +0,0 @@ -// QA Lab WhatsApp user-path action and inbound media scenarios. -import { randomUUID } from "node:crypto"; -import type { WhatsAppQaScenarioDefinition } from "./whatsapp-live.contracts.js"; -import { - WHATSAPP_QA_AUDIO_OGG_OPUS_MIME, - WHATSAPP_QA_AUDIO_TRANSCRIPT_MARKER, - WHATSAPP_QA_ONE_PIXEL_PNG, - assertWhatsAppMessageFromSutPhone, - callWhatsAppGatewaySend, - createWhatsAppQaAudioOggOpusBuffer, - createWhatsAppQaAudioWavBuffer, - createWhatsAppQaPdfBuffer, - matchesWhatsAppSutReactionToTrigger, - waitForNoWhatsAppReply, - waitForScenarioObservedMessage, - waitForWhatsAppScenarioSutMessage, - waitForWhatsAppSutReactionToTrigger, - writeWhatsAppQaWorkspaceFixture, -} from "./whatsapp-live.operations.js"; - -export const WHATSAPP_QA_USER_PATH_SCENARIOS: WhatsAppQaScenarioDefinition[] = [ - { - id: "whatsapp-agent-message-action-react", - title: "WhatsApp user-path agent reaction uses the message tool", - timeoutMs: 90_000, - configOverrides: { - actions: true, - }, - buildRun: () => { - const token = `WHATSAPP_QA_AGENT_REACT_${randomUUID().slice(0, 8).toUpperCase()}`; - return { - afterSend: async (context) => { - const reaction = await waitForWhatsAppSutReactionToTrigger(context, { - expectation: { emoji: "👍" }, - timeoutMs: 60_000, - }); - return `agent message reaction ${reaction.reaction?.emoji ?? ""} observed`; - }, - allowQuietWindowMessage: (message, context) => - matchesWhatsAppSutReactionToTrigger(message, context, { emoji: "👍" }), - configMode: "allowlist", - expectReply: false, - input: - `React to this WhatsApp message with thumbs up for QA action check ${token}. ` + - "Do not send any visible text reply after the reaction.", - matchText: token, - quietWindowMs: 8_000, - target: "dm", - }; - }, - }, - { - id: "whatsapp-agent-message-action-upload-file", - title: "WhatsApp user-path agent upload-file sends media", - timeoutMs: 90_000, - configOverrides: { - actions: true, - }, - buildRun: () => { - const token = `WHATSAPP_QA_AGENT_UPLOAD_${randomUUID().slice(0, 8).toUpperCase()}`; - return { - afterSend: async (context) => { - const media = await waitForScenarioObservedMessage(context, { - observedAfter: context.requestStartedAt, - timeoutMs: 60_000, - match: (message) => - message.kind === "media" && - message.hasMedia === true && - message.mediaType?.startsWith("image/") === true && - message.text.includes(token), - }); - return `agent upload-file media ${media.mediaType ?? ""} observed`; - }, - allowQuietWindowMessage: (message) => - message.kind === "media" && - message.mediaType?.startsWith("image/") === true && - message.text.includes(token), - configMode: "allowlist", - expectReply: false, - input: - `Use the WhatsApp message tool upload-file action to send a PNG with caption ${token}. ` + - "Do not send any visible text reply after the upload.", - matchText: token, - quietWindowMs: 8_000, - target: "dm", - }; - }, - }, - { - id: "whatsapp-group-agent-message-action-react", - title: "WhatsApp group user-path agent reaction uses the message tool", - timeoutMs: 90_000, - configOverrides: { - actions: true, - }, - requiresGroupJid: true, - buildRun: () => { - const token = `WHATSAPP_QA_GROUP_AGENT_REACT_${randomUUID().slice(0, 8).toUpperCase()}`; - return { - afterSend: async (context) => { - const reaction = await waitForWhatsAppSutReactionToTrigger(context, { - expectation: { emoji: "👍" }, - timeoutMs: 60_000, - }); - return `group agent message reaction ${reaction.reaction?.emoji ?? ""} observed`; - }, - allowQuietWindowMessage: (message, context) => - matchesWhatsAppSutReactionToTrigger(message, context, { emoji: "👍" }), - configMode: "allowlist", - expectReply: false, - input: - `openclawqa react to this WhatsApp group message with thumbs up for QA action check ${token}. ` + - "Do not send any visible text reply after the reaction.", - matchText: token, - quietWindowMs: 8_000, - target: "group", - }; - }, - }, - { - id: "whatsapp-group-agent-message-action-upload-file", - title: "WhatsApp group user-path agent upload-file sends media", - timeoutMs: 90_000, - configOverrides: { - actions: true, - }, - requiresGroupJid: true, - buildRun: () => { - const token = `WHATSAPP_QA_GROUP_AGENT_UPLOAD_${randomUUID().slice(0, 8).toUpperCase()}`; - return { - afterSend: async (context) => { - const media = await waitForWhatsAppScenarioSutMessage(context, { - observedAfter: context.requestStartedAt, - targetKind: "group", - timeoutMs: 60_000, - match: (message) => - message.kind === "media" && - message.hasMedia === true && - message.mediaType?.startsWith("image/") === true && - message.text.includes(token), - }); - return `group agent upload-file media ${media.mediaType ?? ""} observed`; - }, - allowQuietWindowMessage: (message) => - message.kind === "media" && - message.mediaType?.startsWith("image/") === true && - message.text.includes(token), - configMode: "allowlist", - expectReply: false, - input: - `openclawqa use the WhatsApp message tool upload-file action to send a PNG with caption ${token}. ` + - "Do not send any visible text reply after the upload.", - matchText: token, - quietWindowMs: 8_000, - target: "group", - }; - }, - }, - { - id: "whatsapp-inbound-reaction-no-trigger", - title: "WhatsApp inbound user reaction does not start a fresh run", - timeoutMs: 90_000, - buildRun: () => { - const token = `WHATSAPP_QA_INBOUND_REACTION_${randomUUID().slice(0, 8).toUpperCase()}`; - return { - afterReply: async (reply, context) => { - assertWhatsAppMessageFromSutPhone(reply, context); - if (!reply.messageId) { - throw new Error("WhatsApp SUT reply did not include a message id to react to."); - } - const reactionStartedAt = new Date(); - await context.driver.sendReaction(context.target, reply.messageId, "❤️", { - fromMe: false, - }); - await waitForNoWhatsAppReply({ - driver: context.driver, - observedAfter: reactionStartedAt, - sutPhoneE164: context.sutPhoneE164, - target: "dm", - windowMs: 5_000, - }); - return "driver reaction to SUT message did not trigger a fresh reply"; - }, - configMode: "allowlist", - expectReply: true, - input: `Reply with only this exact marker before inbound reaction check: ${token}`, - matchText: token, - target: "dm", - }; - }, - }, - { - id: "whatsapp-reply-context-isolation", - title: "WhatsApp direct Gateway send does not reuse prior quote context", - timeoutMs: 120_000, - buildRun: () => { - const token = `WHATSAPP_QA_REPLY_ISOLATION_${randomUUID().slice(0, 8).toUpperCase()}`; - return { - afterReply: async (_reply, context) => { - if (!context.sent.messageId) { - throw new Error("WhatsApp driver did not return a triggering message id."); - } - const quotedStartedAt = new Date(); - await callWhatsAppGatewaySend(context, { - label: "quoted", - message: `${token}_QUOTED`, - replyToId: context.sent.messageId, - }); - await waitForScenarioObservedMessage(context, { - observedAfter: quotedStartedAt, - diagnosticChecks: [ - { - label: "textMarker", - match: (message) => message.text.includes(`${token}_QUOTED`), - }, - { - label: "quotedMessageIdMatchesTrigger", - match: (message) => message.quoted?.messageId === context.sent.messageId, - }, - ], - match: (message) => - message.text.includes(`${token}_QUOTED`) && - message.quoted?.messageId === context.sent.messageId, - }); - - const freshStartedAt = new Date(); - await callWhatsAppGatewaySend(context, { - label: "fresh", - message: `${token}_FRESH`, - }); - const fresh = await waitForScenarioObservedMessage(context, { - observedAfter: freshStartedAt, - match: (message) => message.text.includes(`${token}_FRESH`), - }); - if (fresh.quoted?.messageId) { - throw new Error( - `expected fresh WhatsApp send without quote metadata, got quoted message ${fresh.quoted.messageId}`, - ); - } - return "quoted send and fresh send used independent reply context"; - }, - configMode: "allowlist", - expectReply: true, - input: `Reply with only this exact marker before reply isolation checks: ${token}`, - matchText: token, - target: "dm", - }; - }, - }, - { - id: "whatsapp-inbound-image-caption", - title: "WhatsApp inbound image caption reaches the agent", - timeoutMs: 60_000, - buildRun: () => { - const token = `WHATSAPP_QA_IMAGE_${randomUUID().slice(0, 8).toUpperCase()}`; - return { - configMode: "allowlist", - expectReply: true, - input: `This image caption asks you to reply with only this exact marker: ${token}`, - matchText: token, - sendMode: { - fileName: "whatsapp-qa.png", - kind: "media", - mediaBuffer: WHATSAPP_QA_ONE_PIXEL_PNG, - mediaType: "image/png", - }, - target: "dm", - }; - }, - }, - { - id: "whatsapp-audio-preflight", - title: "WhatsApp inbound audio preflight transcript reaches the agent", - timeoutMs: 90_000, - configOverrides: { - audioPreflight: true, - }, - requiredPluginIds: ["openai"], - buildRun: () => ({ - configMode: "allowlist", - expectReply: true, - input: "", - matchText: WHATSAPP_QA_AUDIO_TRANSCRIPT_MARKER, - sendMode: { - fileName: "whatsapp-qa-audio.ogg", - kind: "media", - mediaBuffer: createWhatsAppQaAudioOggOpusBuffer(), - mediaType: WHATSAPP_QA_AUDIO_OGG_OPUS_MIME, - }, - target: "dm", - }), - }, - { - id: "whatsapp-outbound-media-matrix", - title: "WhatsApp direct Gateway send delivers outbound media variants", - timeoutMs: 120_000, - buildRun: () => { - const token = `WHATSAPP_QA_OUTBOUND_MEDIA_${randomUUID().slice(0, 8).toUpperCase()}`; - return { - afterReply: async (_reply, context) => { - const mediaRootToken = randomUUID().slice(0, 8); - const imagePath = await writeWhatsAppQaWorkspaceFixture(context, { - buffer: WHATSAPP_QA_ONE_PIXEL_PNG, - fileName: `whatsapp-qa-${mediaRootToken}.png`, - }); - const documentPath = await writeWhatsAppQaWorkspaceFixture(context, { - buffer: createWhatsAppQaPdfBuffer(), - fileName: `whatsapp-qa-${mediaRootToken}.pdf`, - }); - const audioPath = await writeWhatsAppQaWorkspaceFixture(context, { - buffer: createWhatsAppQaAudioWavBuffer(), - fileName: `whatsapp-qa-${mediaRootToken}.wav`, - }); - - const imageStartedAt = new Date(); - await callWhatsAppGatewaySend(context, { - label: "image", - mediaUrl: imagePath, - message: `${token}_IMAGE`, - }); - await waitForScenarioObservedMessage(context, { - observedAfter: imageStartedAt, - match: (message) => - message.kind === "media" && - message.hasMedia === true && - message.mediaType?.startsWith("image/") === true && - message.text.includes(`${token}_IMAGE`), - }); - - const documentStartedAt = new Date(); - await callWhatsAppGatewaySend(context, { - forceDocument: true, - label: "document", - mediaUrl: documentPath, - message: `${token}_DOCUMENT`, - }); - await waitForScenarioObservedMessage(context, { - observedAfter: documentStartedAt, - match: (message) => - message.kind === "media" && - message.hasMedia === true && - (message.mediaType === "application/pdf" || - message.mediaFileName?.endsWith(".pdf") === true) && - message.text.includes(`${token}_DOCUMENT`), - }); - - const audioStartedAt = new Date(); - await callWhatsAppGatewaySend(context, { - asVoice: true, - label: "audio", - mediaUrl: audioPath, - message: `${token}_AUDIO`, - }); - await waitForScenarioObservedMessage(context, { - observedAfter: audioStartedAt, - match: (message) => - message.kind === "media" && - message.hasMedia === true && - message.mediaType?.startsWith("audio/") === true, - }); - await waitForScenarioObservedMessage(context, { - observedAfter: audioStartedAt, - match: (message) => message.text.includes(`${token}_AUDIO`), - }); - - const multiStartedAt = new Date(); - await callWhatsAppGatewaySend(context, { - label: "multi", - mediaUrls: [imagePath, documentPath], - message: `${token}_MULTI`, - }); - await waitForScenarioObservedMessage(context, { - observedAfter: multiStartedAt, - match: (message) => - message.kind === "media" && message.mediaType?.startsWith("image/") === true, - }); - await waitForScenarioObservedMessage(context, { - observedAfter: multiStartedAt, - match: (message) => - message.kind === "media" && - (message.mediaType === "application/pdf" || - message.mediaFileName?.endsWith(".pdf") === true), - }); - return "gateway send delivered image, document, audio, and multi-media"; - }, - configMode: "allowlist", - expectReply: true, - input: `Reply with only this exact marker before outbound media checks: ${token}`, - matchText: token, - target: "dm", - }; - }, - }, -]; diff --git a/extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.ts b/extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.ts index 04dbab66e065..6fe0b178a71b 100644 --- a/extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.ts +++ b/extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.ts @@ -8,7 +8,7 @@ import { acquireQaCredentialLease, startQaCredentialLeaseHeartbeat, } from "../live-transports/shared/credential-lease.runtime.js"; -import { listSlackQaScenarioCatalog } from "../live-transports/slack/slack-live.scenarios.js"; +import { resolveSlackQaScenarioIds } from "../live-transports/slack/scenario-selection.js"; import { isTruthyOptIn, trimToValue } from "../mantis-options.runtime.js"; import { createPhaseTimer, type MantisPhaseTimings } from "../mantis-phase-timer.runtime.js"; import { @@ -214,12 +214,9 @@ function resolveScenarioIds(params: { ].join(", ")}. Unsupported: ${unsupported.join(", ")}.`, ); } - const requested = new Set(scenarioIds); - // Slack selects scenarios from catalog order, not CLI order. The watcher - // must mirror that order or both sides can block on different checkpoints. - return listSlackQaScenarioCatalog() - .map((scenario) => scenario.id) - .filter((scenarioId) => requested.has(scenarioId)); + // Mirror the YAML catalog order used by the Slack runner so the watcher + // and runner cannot block on different approval checkpoints. + return resolveSlackQaScenarioIds({ scenarioIds }); } return scenarioIds; } diff --git a/extensions/qa-lab/src/scenario-module-flow.test.ts b/extensions/qa-lab/src/scenario-module-flow.test.ts new file mode 100644 index 000000000000..85c5978e1167 --- /dev/null +++ b/extensions/qa-lab/src/scenario-module-flow.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from "vitest"; +import { qaScenarioModuleFlow } from "./scenario-module-flow.js"; + +describe("QA scenario module flow", () => { + it("resolves a module export argument against the loaded scenario module", () => { + const flow = qaScenarioModuleFlow.moduleSchema.parse({ + module: "./scenario-runtime.js", + call: "runScenario", + args: [{ expr: "scenarioContext" }, { moduleExport: "scenarioImplementation" }], + }); + + expect(qaScenarioModuleFlow.resolveFlow(flow, "Scenario title")).toMatchObject({ + steps: [ + { + actions: [ + { + set: "scenarioModule", + value: { expr: 'await qaImport("./scenario-runtime.js")' }, + }, + { + args: [ + { expr: "scenarioContext" }, + { expr: 'scenarioModule["scenarioImplementation"]' }, + ], + call: "scenarioModule.runScenario", + }, + ], + }, + ], + }); + }); + + it("rejects malformed module export arguments", () => { + expect(() => + qaScenarioModuleFlow.moduleSchema.parse({ + module: "./scenario-runtime.js", + call: "runScenario", + args: [{ moduleExport: "" }], + }), + ).toThrow("moduleExport arguments require a non-empty string export name"); + }); +}); diff --git a/extensions/qa-lab/src/scenario-module-flow.ts b/extensions/qa-lab/src/scenario-module-flow.ts index 691064f048fd..42f1ffe56796 100644 --- a/extensions/qa-lab/src/scenario-module-flow.ts +++ b/extensions/qa-lab/src/scenario-module-flow.ts @@ -1,10 +1,29 @@ // QA Lab scenario module references normalize into the canonical flow shape. import { z } from "zod"; +const qaFlowModuleExportArgSchema = z + .object({ + moduleExport: z.string().trim().min(1), + }) + .strict(); +const qaFlowModuleArgSchema = z.unknown().superRefine((arg, ctx) => { + if ( + typeof arg !== "object" || + arg === null || + !("moduleExport" in arg) || + qaFlowModuleExportArgSchema.safeParse(arg).success + ) { + return; + } + ctx.addIssue({ + code: "custom", + message: "moduleExport arguments require a non-empty string export name", + }); +}); const qaFlowModuleSchema = z.object({ module: z.string().trim().min(1), call: z.string().trim().min(1), - args: z.array(z.unknown()).optional(), + args: z.array(qaFlowModuleArgSchema).optional(), }); const qaFlowExecutionShape = { providerMode: z.enum(["aimock", "live-frontier", "mock-openai"]).optional(), @@ -37,6 +56,16 @@ function normalizeQaScenarioFileMetadata< }; } +function resolveQaScenarioModuleArg(arg: unknown) { + const parsed = qaFlowModuleExportArgSchema.safeParse(arg); + if (!parsed.success) { + return arg; + } + return { + expr: `scenarioModule[${JSON.stringify(parsed.data.moduleExport)}]`, + }; +} + function resolveQaScenarioFileFlow( flow: TFlow | QaScenarioModuleFlow | undefined, title: string, @@ -55,7 +84,7 @@ function resolveQaScenarioFileFlow( }, { call: `scenarioModule.${flow.call}`, - ...(flow.args ? { args: flow.args } : {}), + ...(flow.args ? { args: flow.args.map(resolveQaScenarioModuleArg) } : {}), saveAs: "result", }, ], diff --git a/extensions/qa-lab/src/suite-runtime-flow.ts b/extensions/qa-lab/src/suite-runtime-flow.ts index 20012c8c196e..64710f0cc43f 100644 --- a/extensions/qa-lab/src/suite-runtime-flow.ts +++ b/extensions/qa-lab/src/suite-runtime-flow.ts @@ -340,6 +340,8 @@ export function createQaSuiteScenarioStepRunner( gateway: env.gateway, outputDir: env.outputDir, primaryModel: env.primaryModel, + scenarioId: scenario.id, + scenarioTitle: scenario.title, timeoutMs: execution.timeoutMs ?? deps.liveTurnTimeoutMs(env, 60_000), waitForConfigRestartSettle: async (options) => await waitForConfigRestartSettle(env, options?.restartDelayMs, options?.timeoutMs), diff --git a/extensions/qa-lab/src/suite.test.ts b/extensions/qa-lab/src/suite.test.ts index 0d2a99ed1ac5..8900ecd59b39 100644 --- a/extensions/qa-lab/src/suite.test.ts +++ b/extensions/qa-lab/src/suite.test.ts @@ -165,6 +165,8 @@ describe("qa suite", () => { config: { expected: "value" }, gateway: env.gateway, outputDir: "/tmp/qa-output", + scenarioId: "matrix-preparation-failure", + scenarioTitle: "matrix-preparation-failure", timeoutMs: 45_000, waitForConfigRestartSettle: expect.any(Function), }); diff --git a/qa/scenarios/channels/discord-canary.yaml b/qa/scenarios/channels/discord-canary.yaml index 29d289a8c64f..0719f5a3c757 100644 --- a/qa/scenarios/channels/discord-canary.yaml +++ b/qa/scenarios/channels/discord-canary.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 45000 retryCount: 1 suiteIsolation: isolated - config: { discordScenarioId: discord-canary } flow: module: ./live-transports/discord/scenario-runtime.js - call: runDiscordCanaryScenario - args: [{ expr: discordScenarioContext }] + call: runDiscordScenario + args: + - { expr: discordScenarioContext } + - { moduleExport: discordQaCanaryScenario } diff --git a/qa/scenarios/channels/discord-mention-gating.yaml b/qa/scenarios/channels/discord-mention-gating.yaml index f0d85f6233a2..e0df973743ae 100644 --- a/qa/scenarios/channels/discord-mention-gating.yaml +++ b/qa/scenarios/channels/discord-mention-gating.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 8000 retryCount: 1 suiteIsolation: isolated - config: { discordScenarioId: discord-mention-gating } flow: module: ./live-transports/discord/scenario-runtime.js - call: runDiscordMentionGatingScenario - args: [{ expr: discordScenarioContext }] + call: runDiscordScenario + args: + - { expr: discordScenarioContext } + - { moduleExport: discordQaMentionGatingScenario } diff --git a/qa/scenarios/channels/discord-native-help-command-registration.yaml b/qa/scenarios/channels/discord-native-help-command-registration.yaml index 68acac3a51d7..1c9c5b3a81bb 100644 --- a/qa/scenarios/channels/discord-native-help-command-registration.yaml +++ b/qa/scenarios/channels/discord-native-help-command-registration.yaml @@ -10,8 +10,9 @@ scenario: timeoutMs: 45000 retryCount: 1 suiteIsolation: isolated - config: { discordScenarioId: discord-native-help-command-registration } flow: module: ./live-transports/discord/scenario-runtime.js - call: runDiscordNativeHelpCommandRegistrationScenario - args: [{ expr: discordScenarioContext }] + call: runDiscordScenario + args: + - { expr: discordScenarioContext } + - { moduleExport: discordQaNativeHelpCommandRegistrationScenario } diff --git a/qa/scenarios/channels/discord-status-reactions-tool-only.yaml b/qa/scenarios/channels/discord-status-reactions-tool-only.yaml index 3747b21c3e23..4c50d09c3054 100644 --- a/qa/scenarios/channels/discord-status-reactions-tool-only.yaml +++ b/qa/scenarios/channels/discord-status-reactions-tool-only.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 75000 retryCount: 1 suiteIsolation: isolated - config: { discordScenarioId: discord-status-reactions-tool-only } flow: module: ./live-transports/discord/scenario-runtime.js - call: runDiscordStatusReactionsToolOnlyScenario - args: [{ expr: discordScenarioContext }] + call: runDiscordScenario + args: + - { expr: discordScenarioContext } + - { moduleExport: discordQaStatusReactionsToolOnlyScenario } diff --git a/qa/scenarios/channels/discord-thread-reply-filepath-attachment.yaml b/qa/scenarios/channels/discord-thread-reply-filepath-attachment.yaml index 5ded427da541..07109ad0837d 100644 --- a/qa/scenarios/channels/discord-thread-reply-filepath-attachment.yaml +++ b/qa/scenarios/channels/discord-thread-reply-filepath-attachment.yaml @@ -12,8 +12,9 @@ scenario: timeoutMs: 45000 retryCount: 1 suiteIsolation: isolated - config: { discordScenarioId: discord-thread-reply-filepath-attachment } flow: module: ./live-transports/discord/scenario-runtime.js - call: runDiscordThreadReplyFilepathAttachmentScenario - args: [{ expr: discordScenarioContext }] + call: runDiscordScenario + args: + - { expr: discordScenarioContext } + - { moduleExport: discordQaThreadReplyFilepathAttachmentScenario } diff --git a/qa/scenarios/channels/discord-voice-autojoin.yaml b/qa/scenarios/channels/discord-voice-autojoin.yaml index b2927b23bf12..7c6e354a4ddf 100644 --- a/qa/scenarios/channels/discord-voice-autojoin.yaml +++ b/qa/scenarios/channels/discord-voice-autojoin.yaml @@ -10,8 +10,9 @@ scenario: timeoutMs: 60000 retryCount: 1 suiteIsolation: isolated - config: { discordScenarioId: discord-voice-autojoin } flow: module: ./live-transports/discord/scenario-runtime.js - call: runDiscordVoiceAutojoinScenario - args: [{ expr: discordScenarioContext }] + call: runDiscordScenario + args: + - { expr: discordScenarioContext } + - { moduleExport: discordQaVoiceAutojoinScenario } diff --git a/qa/scenarios/channels/slack-allowlist-block.yaml b/qa/scenarios/channels/slack-allowlist-block.yaml index cd8a8b011a88..10644a950f2b 100644 --- a/qa/scenarios/channels/slack-allowlist-block.yaml +++ b/qa/scenarios/channels/slack-allowlist-block.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 8000 retryCount: 1 suiteIsolation: isolated - config: { slackScenarioId: slack-allowlist-block } flow: module: ./live-transports/slack/scenario-runtime.js - call: runSlackAllowlistBlockScenario - args: [{ expr: slackScenarioContext }] + call: runSlackScenario + args: + - { expr: slackScenarioContext } + - { moduleExport: slackQaAllowlistBlockScenario } diff --git a/qa/scenarios/channels/slack-approval-exec-native.yaml b/qa/scenarios/channels/slack-approval-exec-native.yaml index 1efa21acda66..4da0d94de1a9 100644 --- a/qa/scenarios/channels/slack-approval-exec-native.yaml +++ b/qa/scenarios/channels/slack-approval-exec-native.yaml @@ -13,8 +13,9 @@ scenario: timeoutMs: 60000 retryCount: 1 suiteIsolation: isolated - config: { slackScenarioId: slack-approval-exec-native } flow: module: ./live-transports/slack/scenario-runtime.js - call: runSlackApprovalExecNativeScenario - args: [{ expr: slackScenarioContext }] + call: runSlackScenario + args: + - { expr: slackScenarioContext } + - { moduleExport: slackQaApprovalExecNativeScenario } diff --git a/qa/scenarios/channels/slack-approval-plugin-native.yaml b/qa/scenarios/channels/slack-approval-plugin-native.yaml index 426d0cdf15c5..b6c53ff3f46f 100644 --- a/qa/scenarios/channels/slack-approval-plugin-native.yaml +++ b/qa/scenarios/channels/slack-approval-plugin-native.yaml @@ -13,8 +13,9 @@ scenario: timeoutMs: 60000 retryCount: 1 suiteIsolation: isolated - config: { slackScenarioId: slack-approval-plugin-native } flow: module: ./live-transports/slack/scenario-runtime.js - call: runSlackApprovalPluginNativeScenario - args: [{ expr: slackScenarioContext }] + call: runSlackScenario + args: + - { expr: slackScenarioContext } + - { moduleExport: slackQaApprovalPluginNativeScenario } diff --git a/qa/scenarios/channels/slack-canary.yaml b/qa/scenarios/channels/slack-canary.yaml index 8a63de0552a8..cd55fb6b01ff 100644 --- a/qa/scenarios/channels/slack-canary.yaml +++ b/qa/scenarios/channels/slack-canary.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 45000 retryCount: 1 suiteIsolation: isolated - config: { slackScenarioId: slack-canary } flow: module: ./live-transports/slack/scenario-runtime.js - call: runSlackCanaryScenario - args: [{ expr: slackScenarioContext }] + call: runSlackScenario + args: + - { expr: slackScenarioContext } + - { moduleExport: slackQaCanaryScenario } diff --git a/qa/scenarios/channels/slack-channel-disabled-warning.yaml b/qa/scenarios/channels/slack-channel-disabled-warning.yaml index 4fbc6c0adfc4..1bc056666f5e 100644 --- a/qa/scenarios/channels/slack-channel-disabled-warning.yaml +++ b/qa/scenarios/channels/slack-channel-disabled-warning.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 8000 retryCount: 1 suiteIsolation: isolated - config: { slackScenarioId: slack-channel-disabled-warning } flow: module: ./live-transports/slack/scenario-runtime.js - call: runSlackChannelDisabledWarningScenario - args: [{ expr: slackScenarioContext }] + call: runSlackScenario + args: + - { expr: slackScenarioContext } + - { moduleExport: slackQaChannelDisabledWarningScenario } diff --git a/qa/scenarios/channels/slack-chart-presentation-native.yaml b/qa/scenarios/channels/slack-chart-presentation-native.yaml index b2324d0962f2..cb9c1c5af162 100644 --- a/qa/scenarios/channels/slack-chart-presentation-native.yaml +++ b/qa/scenarios/channels/slack-chart-presentation-native.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 90000 retryCount: 1 suiteIsolation: isolated - config: { slackScenarioId: slack-chart-presentation-native } flow: module: ./live-transports/slack/scenario-runtime.js - call: runSlackChartPresentationNativeScenario - args: [{ expr: slackScenarioContext }] + call: runSlackScenario + args: + - { expr: slackScenarioContext } + - { moduleExport: slackQaChartPresentationNativeScenario } diff --git a/qa/scenarios/channels/slack-codex-approval-exec-native.yaml b/qa/scenarios/channels/slack-codex-approval-exec-native.yaml index c9283db3541b..4988c5db0f52 100644 --- a/qa/scenarios/channels/slack-codex-approval-exec-native.yaml +++ b/qa/scenarios/channels/slack-codex-approval-exec-native.yaml @@ -14,8 +14,9 @@ scenario: timeoutMs: 180000 retryCount: 1 suiteIsolation: isolated - config: { slackScenarioId: slack-codex-approval-exec-native } flow: module: ./live-transports/slack/scenario-runtime.js - call: runSlackCodexApprovalExecNativeScenario - args: [{ expr: slackScenarioContext }] + call: runSlackScenario + args: + - { expr: slackScenarioContext } + - { moduleExport: slackQaCodexApprovalExecNativeScenario } diff --git a/qa/scenarios/channels/slack-codex-approval-plugin-native.yaml b/qa/scenarios/channels/slack-codex-approval-plugin-native.yaml index 4a426413385f..7518f5e55825 100644 --- a/qa/scenarios/channels/slack-codex-approval-plugin-native.yaml +++ b/qa/scenarios/channels/slack-codex-approval-plugin-native.yaml @@ -14,8 +14,9 @@ scenario: timeoutMs: 180000 retryCount: 1 suiteIsolation: isolated - config: { slackScenarioId: slack-codex-approval-plugin-native } flow: module: ./live-transports/slack/scenario-runtime.js - call: runSlackCodexApprovalPluginNativeScenario - args: [{ expr: slackScenarioContext }] + call: runSlackScenario + args: + - { expr: slackScenarioContext } + - { moduleExport: slackQaCodexApprovalPluginNativeScenario } diff --git a/qa/scenarios/channels/slack-mention-gating.yaml b/qa/scenarios/channels/slack-mention-gating.yaml index f37cb5147bbe..5ff2a1474fcc 100644 --- a/qa/scenarios/channels/slack-mention-gating.yaml +++ b/qa/scenarios/channels/slack-mention-gating.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 8000 retryCount: 1 suiteIsolation: isolated - config: { slackScenarioId: slack-mention-gating } flow: module: ./live-transports/slack/scenario-runtime.js - call: runSlackMentionGatingScenario - args: [{ expr: slackScenarioContext }] + call: runSlackScenario + args: + - { expr: slackScenarioContext } + - { moduleExport: slackQaMentionGatingScenario } diff --git a/qa/scenarios/channels/slack-mpim-app-mention-dedupe.yaml b/qa/scenarios/channels/slack-mpim-app-mention-dedupe.yaml index 59ccdfe55aa2..c09cfa438a23 100644 --- a/qa/scenarios/channels/slack-mpim-app-mention-dedupe.yaml +++ b/qa/scenarios/channels/slack-mpim-app-mention-dedupe.yaml @@ -13,8 +13,9 @@ scenario: timeoutMs: 180000 retryCount: 0 suiteIsolation: isolated - config: { slackScenarioId: slack-mpim-app-mention-dedupe } flow: module: ./live-transports/slack/scenario-runtime.js - call: runSlackMpimAppMentionDedupeScenario - args: [{ expr: slackScenarioContext }] + call: runSlackScenario + args: + - { expr: slackScenarioContext } + - { moduleExport: slackQaMpimAppMentionDedupeScenario } diff --git a/qa/scenarios/channels/slack-progress-commentary-false.yaml b/qa/scenarios/channels/slack-progress-commentary-false.yaml index 259131af2198..9c51196069ee 100644 --- a/qa/scenarios/channels/slack-progress-commentary-false.yaml +++ b/qa/scenarios/channels/slack-progress-commentary-false.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 90000 retryCount: 1 suiteIsolation: isolated - config: { slackScenarioId: slack-progress-commentary-false } flow: module: ./live-transports/slack/scenario-runtime.js - call: runSlackProgressCommentaryFalseScenario - args: [{ expr: slackScenarioContext }] + call: runSlackScenario + args: + - { expr: slackScenarioContext } + - { moduleExport: slackQaProgressCommentaryFalseScenario } diff --git a/qa/scenarios/channels/slack-progress-commentary-omitted.yaml b/qa/scenarios/channels/slack-progress-commentary-omitted.yaml index 13c73b57b067..aa7d9dd7b4ae 100644 --- a/qa/scenarios/channels/slack-progress-commentary-omitted.yaml +++ b/qa/scenarios/channels/slack-progress-commentary-omitted.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 90000 retryCount: 1 suiteIsolation: isolated - config: { slackScenarioId: slack-progress-commentary-omitted } flow: module: ./live-transports/slack/scenario-runtime.js - call: runSlackProgressCommentaryOmittedScenario - args: [{ expr: slackScenarioContext }] + call: runSlackScenario + args: + - { expr: slackScenarioContext } + - { moduleExport: slackQaProgressCommentaryOmittedScenario } diff --git a/qa/scenarios/channels/slack-progress-commentary-true.yaml b/qa/scenarios/channels/slack-progress-commentary-true.yaml index c9818477a568..357a87d86863 100644 --- a/qa/scenarios/channels/slack-progress-commentary-true.yaml +++ b/qa/scenarios/channels/slack-progress-commentary-true.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 90000 retryCount: 1 suiteIsolation: isolated - config: { slackScenarioId: slack-progress-commentary-true } flow: module: ./live-transports/slack/scenario-runtime.js - call: runSlackProgressCommentaryTrueScenario - args: [{ expr: slackScenarioContext }] + call: runSlackScenario + args: + - { expr: slackScenarioContext } + - { moduleExport: slackQaProgressCommentaryTrueScenario } diff --git a/qa/scenarios/channels/slack-progress-commentary-verbose-dedupe.yaml b/qa/scenarios/channels/slack-progress-commentary-verbose-dedupe.yaml index aa29c167ff1f..ce6b3547a8e1 100644 --- a/qa/scenarios/channels/slack-progress-commentary-verbose-dedupe.yaml +++ b/qa/scenarios/channels/slack-progress-commentary-verbose-dedupe.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 90000 retryCount: 1 suiteIsolation: isolated - config: { slackScenarioId: slack-progress-commentary-verbose-dedupe } flow: module: ./live-transports/slack/scenario-runtime.js - call: runSlackProgressCommentaryVerboseDedupeScenario - args: [{ expr: slackScenarioContext }] + call: runSlackScenario + args: + - { expr: slackScenarioContext } + - { moduleExport: slackQaProgressCommentaryVerboseDedupeScenario } diff --git a/qa/scenarios/channels/slack-reaction-glyph-native.yaml b/qa/scenarios/channels/slack-reaction-glyph-native.yaml index bffa256f63d1..bffb4ff1a8d8 100644 --- a/qa/scenarios/channels/slack-reaction-glyph-native.yaml +++ b/qa/scenarios/channels/slack-reaction-glyph-native.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 90000 retryCount: 1 suiteIsolation: isolated - config: { slackScenarioId: slack-reaction-glyph-native } flow: module: ./live-transports/slack/scenario-runtime.js - call: runSlackReactionGlyphNativeScenario - args: [{ expr: slackScenarioContext }] + call: runSlackScenario + args: + - { expr: slackScenarioContext } + - { moduleExport: slackQaReactionGlyphNativeScenario } diff --git a/qa/scenarios/channels/slack-table-invalid-blocks-fallback.yaml b/qa/scenarios/channels/slack-table-invalid-blocks-fallback.yaml index dc8eff525aa5..605e0d8adeb6 100644 --- a/qa/scenarios/channels/slack-table-invalid-blocks-fallback.yaml +++ b/qa/scenarios/channels/slack-table-invalid-blocks-fallback.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 45000 retryCount: 1 suiteIsolation: isolated - config: { slackScenarioId: slack-table-invalid-blocks-fallback } flow: module: ./live-transports/slack/scenario-runtime.js - call: runSlackTableInvalidBlocksFallbackScenario - args: [{ expr: slackScenarioContext }] + call: runSlackScenario + args: + - { expr: slackScenarioContext } + - { moduleExport: slackQaTableInvalidBlocksFallbackScenario } diff --git a/qa/scenarios/channels/slack-table-presentation-native.yaml b/qa/scenarios/channels/slack-table-presentation-native.yaml index da6659c81a4d..21bc6f02ac5f 100644 --- a/qa/scenarios/channels/slack-table-presentation-native.yaml +++ b/qa/scenarios/channels/slack-table-presentation-native.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 90000 retryCount: 1 suiteIsolation: isolated - config: { slackScenarioId: slack-table-presentation-native } flow: module: ./live-transports/slack/scenario-runtime.js - call: runSlackTablePresentationNativeScenario - args: [{ expr: slackScenarioContext }] + call: runSlackScenario + args: + - { expr: slackScenarioContext } + - { moduleExport: slackQaTablePresentationNativeScenario } diff --git a/qa/scenarios/channels/slack-top-level-reply-shape.yaml b/qa/scenarios/channels/slack-top-level-reply-shape.yaml index cdfba1c9a0d7..226397ac5f23 100644 --- a/qa/scenarios/channels/slack-top-level-reply-shape.yaml +++ b/qa/scenarios/channels/slack-top-level-reply-shape.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 45000 retryCount: 1 suiteIsolation: isolated - config: { slackScenarioId: slack-top-level-reply-shape } flow: module: ./live-transports/slack/scenario-runtime.js - call: runSlackTopLevelReplyShapeScenario - args: [{ expr: slackScenarioContext }] + call: runSlackScenario + args: + - { expr: slackScenarioContext } + - { moduleExport: slackQaTopLevelReplyShapeScenario } diff --git a/qa/scenarios/channels/whatsapp-agent-message-action-react.yaml b/qa/scenarios/channels/whatsapp-agent-message-action-react.yaml index 82733ae573d1..d3b551a51011 100644 --- a/qa/scenarios/channels/whatsapp-agent-message-action-react.yaml +++ b/qa/scenarios/channels/whatsapp-agent-message-action-react.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 90000 retryCount: 1 suiteIsolation: isolated - config: { whatsappScenarioId: whatsapp-agent-message-action-react } flow: module: ./live-transports/whatsapp/scenario-runtime.js - call: runWhatsAppAgentMessageActionReactScenario - args: [{ expr: whatsappScenarioContext }] + call: runWhatsAppScenario + args: + - { expr: whatsappScenarioContext } + - { moduleExport: whatsappQaAgentMessageActionReactScenario } diff --git a/qa/scenarios/channels/whatsapp-agent-message-action-upload-file.yaml b/qa/scenarios/channels/whatsapp-agent-message-action-upload-file.yaml index 1a5d307c5971..ebe1665df70f 100644 --- a/qa/scenarios/channels/whatsapp-agent-message-action-upload-file.yaml +++ b/qa/scenarios/channels/whatsapp-agent-message-action-upload-file.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 90000 retryCount: 1 suiteIsolation: isolated - config: { whatsappScenarioId: whatsapp-agent-message-action-upload-file } flow: module: ./live-transports/whatsapp/scenario-runtime.js - call: runWhatsAppAgentMessageActionUploadFileScenario - args: [{ expr: whatsappScenarioContext }] + call: runWhatsAppScenario + args: + - { expr: whatsappScenarioContext } + - { moduleExport: whatsappQaAgentMessageActionUploadFileScenario } diff --git a/qa/scenarios/channels/whatsapp-approval-exec-deny-native.yaml b/qa/scenarios/channels/whatsapp-approval-exec-deny-native.yaml index d058ff63a269..4a5eede8fdf4 100644 --- a/qa/scenarios/channels/whatsapp-approval-exec-deny-native.yaml +++ b/qa/scenarios/channels/whatsapp-approval-exec-deny-native.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 60000 retryCount: 1 suiteIsolation: isolated - config: { whatsappScenarioId: whatsapp-approval-exec-deny-native } flow: module: ./live-transports/whatsapp/scenario-runtime.js - call: runWhatsAppApprovalExecDenyNativeScenario - args: [{ expr: whatsappScenarioContext }] + call: runWhatsAppScenario + args: + - { expr: whatsappScenarioContext } + - { moduleExport: whatsappQaApprovalExecDenyNativeScenario } diff --git a/qa/scenarios/channels/whatsapp-approval-exec-group-reaction-native.yaml b/qa/scenarios/channels/whatsapp-approval-exec-group-reaction-native.yaml index 730e2410e806..b5ab5033fb2f 100644 --- a/qa/scenarios/channels/whatsapp-approval-exec-group-reaction-native.yaml +++ b/qa/scenarios/channels/whatsapp-approval-exec-group-reaction-native.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 60000 retryCount: 1 suiteIsolation: isolated - config: { whatsappScenarioId: whatsapp-approval-exec-group-reaction-native } flow: module: ./live-transports/whatsapp/scenario-runtime.js - call: runWhatsAppApprovalExecGroupReactionNativeScenario - args: [{ expr: whatsappScenarioContext }] + call: runWhatsAppScenario + args: + - { expr: whatsappScenarioContext } + - { moduleExport: whatsappQaApprovalExecGroupReactionNativeScenario } diff --git a/qa/scenarios/channels/whatsapp-approval-exec-native.yaml b/qa/scenarios/channels/whatsapp-approval-exec-native.yaml index a8d240583503..0925d111a3af 100644 --- a/qa/scenarios/channels/whatsapp-approval-exec-native.yaml +++ b/qa/scenarios/channels/whatsapp-approval-exec-native.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 60000 retryCount: 1 suiteIsolation: isolated - config: { whatsappScenarioId: whatsapp-approval-exec-native } flow: module: ./live-transports/whatsapp/scenario-runtime.js - call: runWhatsAppApprovalExecNativeScenario - args: [{ expr: whatsappScenarioContext }] + call: runWhatsAppScenario + args: + - { expr: whatsappScenarioContext } + - { moduleExport: whatsappQaApprovalExecNativeScenario } diff --git a/qa/scenarios/channels/whatsapp-approval-exec-reaction-native.yaml b/qa/scenarios/channels/whatsapp-approval-exec-reaction-native.yaml index 19a45bfa7624..95bf88123c7e 100644 --- a/qa/scenarios/channels/whatsapp-approval-exec-reaction-native.yaml +++ b/qa/scenarios/channels/whatsapp-approval-exec-reaction-native.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 60000 retryCount: 1 suiteIsolation: isolated - config: { whatsappScenarioId: whatsapp-approval-exec-reaction-native } flow: module: ./live-transports/whatsapp/scenario-runtime.js - call: runWhatsAppApprovalExecReactionNativeScenario - args: [{ expr: whatsappScenarioContext }] + call: runWhatsAppScenario + args: + - { expr: whatsappScenarioContext } + - { moduleExport: whatsappQaApprovalExecReactionNativeScenario } diff --git a/qa/scenarios/channels/whatsapp-approval-plugin-native.yaml b/qa/scenarios/channels/whatsapp-approval-plugin-native.yaml index 7b1b3bef42bc..c699e53ec151 100644 --- a/qa/scenarios/channels/whatsapp-approval-plugin-native.yaml +++ b/qa/scenarios/channels/whatsapp-approval-plugin-native.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 60000 retryCount: 1 suiteIsolation: isolated - config: { whatsappScenarioId: whatsapp-approval-plugin-native } flow: module: ./live-transports/whatsapp/scenario-runtime.js - call: runWhatsAppApprovalPluginNativeScenario - args: [{ expr: whatsappScenarioContext }] + call: runWhatsAppScenario + args: + - { expr: whatsappScenarioContext } + - { moduleExport: whatsappQaApprovalPluginNativeScenario } diff --git a/qa/scenarios/channels/whatsapp-audio-preflight.yaml b/qa/scenarios/channels/whatsapp-audio-preflight.yaml index b863ff0ba5ab..3917405db106 100644 --- a/qa/scenarios/channels/whatsapp-audio-preflight.yaml +++ b/qa/scenarios/channels/whatsapp-audio-preflight.yaml @@ -2,6 +2,8 @@ title: WhatsApp inbound audio preflight transcript reaches the agent scenario: id: whatsapp-audio-preflight surface: channels + plugins: + - openai coverage: primary: - whatsapp.inbound-media-download @@ -11,8 +13,9 @@ scenario: timeoutMs: 90000 retryCount: 1 suiteIsolation: isolated - config: { whatsappScenarioId: whatsapp-audio-preflight } flow: module: ./live-transports/whatsapp/scenario-runtime.js - call: runWhatsAppAudioPreflightScenario - args: [{ expr: whatsappScenarioContext }] + call: runWhatsAppScenario + args: + - { expr: whatsappScenarioContext } + - { moduleExport: whatsappQaAudioPreflightScenario } diff --git a/qa/scenarios/channels/whatsapp-broadcast-group-fanout.yaml b/qa/scenarios/channels/whatsapp-broadcast-group-fanout.yaml index ed268b7229f3..7ded0147bd32 100644 --- a/qa/scenarios/channels/whatsapp-broadcast-group-fanout.yaml +++ b/qa/scenarios/channels/whatsapp-broadcast-group-fanout.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 120000 retryCount: 1 suiteIsolation: isolated - config: { whatsappScenarioId: whatsapp-broadcast-group-fanout } flow: module: ./live-transports/whatsapp/scenario-runtime.js - call: runWhatsAppBroadcastGroupFanoutScenario - args: [{ expr: whatsappScenarioContext }] + call: runWhatsAppScenario + args: + - { expr: whatsappScenarioContext } + - { moduleExport: whatsappQaBroadcastGroupFanoutScenario } diff --git a/qa/scenarios/channels/whatsapp-canary.yaml b/qa/scenarios/channels/whatsapp-canary.yaml index 5f59b267d97d..d4af059e9ccb 100644 --- a/qa/scenarios/channels/whatsapp-canary.yaml +++ b/qa/scenarios/channels/whatsapp-canary.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 60000 retryCount: 1 suiteIsolation: isolated - config: { whatsappScenarioId: whatsapp-canary } flow: module: ./live-transports/whatsapp/scenario-runtime.js - call: runWhatsAppCanaryScenario - args: [{ expr: whatsappScenarioContext }] + call: runWhatsAppScenario + args: + - { expr: whatsappScenarioContext } + - { moduleExport: whatsappQaCanaryScenario } diff --git a/qa/scenarios/channels/whatsapp-group-activation-always.yaml b/qa/scenarios/channels/whatsapp-group-activation-always.yaml index ca853a128c41..4750f45b9a22 100644 --- a/qa/scenarios/channels/whatsapp-group-activation-always.yaml +++ b/qa/scenarios/channels/whatsapp-group-activation-always.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 120000 retryCount: 1 suiteIsolation: isolated - config: { whatsappScenarioId: whatsapp-group-activation-always } flow: module: ./live-transports/whatsapp/scenario-runtime.js - call: runWhatsAppGroupActivationAlwaysScenario - args: [{ expr: whatsappScenarioContext }] + call: runWhatsAppScenario + args: + - { expr: whatsappScenarioContext } + - { moduleExport: whatsappQaGroupActivationAlwaysScenario } diff --git a/qa/scenarios/channels/whatsapp-group-agent-message-action-react.yaml b/qa/scenarios/channels/whatsapp-group-agent-message-action-react.yaml index 3a22b9034737..954ebcd30384 100644 --- a/qa/scenarios/channels/whatsapp-group-agent-message-action-react.yaml +++ b/qa/scenarios/channels/whatsapp-group-agent-message-action-react.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 90000 retryCount: 1 suiteIsolation: isolated - config: { whatsappScenarioId: whatsapp-group-agent-message-action-react } flow: module: ./live-transports/whatsapp/scenario-runtime.js - call: runWhatsAppGroupAgentMessageActionReactScenario - args: [{ expr: whatsappScenarioContext }] + call: runWhatsAppScenario + args: + - { expr: whatsappScenarioContext } + - { moduleExport: whatsappQaGroupAgentMessageActionReactScenario } diff --git a/qa/scenarios/channels/whatsapp-group-agent-message-action-upload-file.yaml b/qa/scenarios/channels/whatsapp-group-agent-message-action-upload-file.yaml index f4601c4b250e..0f2e2e9750dc 100644 --- a/qa/scenarios/channels/whatsapp-group-agent-message-action-upload-file.yaml +++ b/qa/scenarios/channels/whatsapp-group-agent-message-action-upload-file.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 90000 retryCount: 1 suiteIsolation: isolated - config: { whatsappScenarioId: whatsapp-group-agent-message-action-upload-file } flow: module: ./live-transports/whatsapp/scenario-runtime.js - call: runWhatsAppGroupAgentMessageActionUploadFileScenario - args: [{ expr: whatsappScenarioContext }] + call: runWhatsAppScenario + args: + - { expr: whatsappScenarioContext } + - { moduleExport: whatsappQaGroupAgentMessageActionUploadFileScenario } diff --git a/qa/scenarios/channels/whatsapp-group-allowlist-block.yaml b/qa/scenarios/channels/whatsapp-group-allowlist-block.yaml index c2d519a634b8..32b72d95c5ad 100644 --- a/qa/scenarios/channels/whatsapp-group-allowlist-block.yaml +++ b/qa/scenarios/channels/whatsapp-group-allowlist-block.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 8000 retryCount: 1 suiteIsolation: isolated - config: { whatsappScenarioId: whatsapp-group-allowlist-block } flow: module: ./live-transports/whatsapp/scenario-runtime.js - call: runWhatsAppGroupAllowlistBlockScenario - args: [{ expr: whatsappScenarioContext }] + call: runWhatsAppScenario + args: + - { expr: whatsappScenarioContext } + - { moduleExport: whatsappQaGroupAllowlistBlockScenario } diff --git a/qa/scenarios/channels/whatsapp-group-audio-gating.yaml b/qa/scenarios/channels/whatsapp-group-audio-gating.yaml index 655cc69b8b00..2931d5e3976e 100644 --- a/qa/scenarios/channels/whatsapp-group-audio-gating.yaml +++ b/qa/scenarios/channels/whatsapp-group-audio-gating.yaml @@ -2,6 +2,8 @@ title: WhatsApp group audio mention gating scenario: id: whatsapp-group-audio-gating surface: channels + plugins: + - openai coverage: primary: - whatsapp.group-allowlists @@ -12,8 +14,9 @@ scenario: timeoutMs: 120000 retryCount: 1 suiteIsolation: isolated - config: { whatsappScenarioId: whatsapp-group-audio-gating } flow: module: ./live-transports/whatsapp/scenario-runtime.js - call: runWhatsAppGroupAudioGatingScenario - args: [{ expr: whatsappScenarioContext }] + call: runWhatsAppScenario + args: + - { expr: whatsappScenarioContext } + - { moduleExport: whatsappQaGroupAudioGatingScenario } diff --git a/qa/scenarios/channels/whatsapp-group-outbound-audio.yaml b/qa/scenarios/channels/whatsapp-group-outbound-audio.yaml index ee82159c40f6..51ef970e59b5 100644 --- a/qa/scenarios/channels/whatsapp-group-outbound-audio.yaml +++ b/qa/scenarios/channels/whatsapp-group-outbound-audio.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 90000 retryCount: 1 suiteIsolation: isolated - config: { whatsappScenarioId: whatsapp-group-outbound-audio } flow: module: ./live-transports/whatsapp/scenario-runtime.js - call: runWhatsAppGroupOutboundAudioScenario - args: [{ expr: whatsappScenarioContext }] + call: runWhatsAppScenario + args: + - { expr: whatsappScenarioContext } + - { moduleExport: whatsappQaGroupOutboundAudioScenario } diff --git a/qa/scenarios/channels/whatsapp-group-outbound-media.yaml b/qa/scenarios/channels/whatsapp-group-outbound-media.yaml index a2380d7d4b5c..ea73e7920c2d 100644 --- a/qa/scenarios/channels/whatsapp-group-outbound-media.yaml +++ b/qa/scenarios/channels/whatsapp-group-outbound-media.yaml @@ -12,8 +12,9 @@ scenario: timeoutMs: 120000 retryCount: 1 suiteIsolation: isolated - config: { whatsappScenarioId: whatsapp-group-outbound-media } flow: module: ./live-transports/whatsapp/scenario-runtime.js - call: runWhatsAppGroupOutboundMediaScenario - args: [{ expr: whatsappScenarioContext }] + call: runWhatsAppScenario + args: + - { expr: whatsappScenarioContext } + - { moduleExport: whatsappQaGroupOutboundMediaScenario } diff --git a/qa/scenarios/channels/whatsapp-group-outbound-poll.yaml b/qa/scenarios/channels/whatsapp-group-outbound-poll.yaml index ec9d03d0341d..e9cba036d642 100644 --- a/qa/scenarios/channels/whatsapp-group-outbound-poll.yaml +++ b/qa/scenarios/channels/whatsapp-group-outbound-poll.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 90000 retryCount: 1 suiteIsolation: isolated - config: { whatsappScenarioId: whatsapp-group-outbound-poll } flow: module: ./live-transports/whatsapp/scenario-runtime.js - call: runWhatsAppGroupOutboundPollScenario - args: [{ expr: whatsappScenarioContext }] + call: runWhatsAppScenario + args: + - { expr: whatsappScenarioContext } + - { moduleExport: whatsappQaGroupOutboundPollScenario } diff --git a/qa/scenarios/channels/whatsapp-group-pending-history-context.yaml b/qa/scenarios/channels/whatsapp-group-pending-history-context.yaml index ba2cd9b47c73..bfc85b974685 100644 --- a/qa/scenarios/channels/whatsapp-group-pending-history-context.yaml +++ b/qa/scenarios/channels/whatsapp-group-pending-history-context.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 90000 retryCount: 1 suiteIsolation: isolated - config: { whatsappScenarioId: whatsapp-group-pending-history-context } flow: module: ./live-transports/whatsapp/scenario-runtime.js - call: runWhatsAppGroupPendingHistoryContextScenario - args: [{ expr: whatsappScenarioContext }] + call: runWhatsAppScenario + args: + - { expr: whatsappScenarioContext } + - { moduleExport: whatsappQaGroupPendingHistoryContextScenario } diff --git a/qa/scenarios/channels/whatsapp-group-reply-to-bot-triggers.yaml b/qa/scenarios/channels/whatsapp-group-reply-to-bot-triggers.yaml index 483abe9ab7b5..74b38d83c3c7 100644 --- a/qa/scenarios/channels/whatsapp-group-reply-to-bot-triggers.yaml +++ b/qa/scenarios/channels/whatsapp-group-reply-to-bot-triggers.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 120000 retryCount: 1 suiteIsolation: isolated - config: { whatsappScenarioId: whatsapp-group-reply-to-bot-triggers } flow: module: ./live-transports/whatsapp/scenario-runtime.js - call: runWhatsAppGroupReplyToBotTriggersScenario - args: [{ expr: whatsappScenarioContext }] + call: runWhatsAppScenario + args: + - { expr: whatsappScenarioContext } + - { moduleExport: whatsappQaGroupReplyToBotTriggersScenario } diff --git a/qa/scenarios/channels/whatsapp-group-reply-to-message.yaml b/qa/scenarios/channels/whatsapp-group-reply-to-message.yaml index e1784521e6c7..a7afdb5adcf0 100644 --- a/qa/scenarios/channels/whatsapp-group-reply-to-message.yaml +++ b/qa/scenarios/channels/whatsapp-group-reply-to-message.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 60000 retryCount: 1 suiteIsolation: isolated - config: { whatsappScenarioId: whatsapp-group-reply-to-message } flow: module: ./live-transports/whatsapp/scenario-runtime.js - call: runWhatsAppGroupReplyToMessageScenario - args: [{ expr: whatsappScenarioContext }] + call: runWhatsAppScenario + args: + - { expr: whatsappScenarioContext } + - { moduleExport: whatsappQaGroupReplyToMessageScenario } diff --git a/qa/scenarios/channels/whatsapp-inbound-image-caption.yaml b/qa/scenarios/channels/whatsapp-inbound-image-caption.yaml index 4702fc390583..1a48465af7b1 100644 --- a/qa/scenarios/channels/whatsapp-inbound-image-caption.yaml +++ b/qa/scenarios/channels/whatsapp-inbound-image-caption.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 60000 retryCount: 1 suiteIsolation: isolated - config: { whatsappScenarioId: whatsapp-inbound-image-caption } flow: module: ./live-transports/whatsapp/scenario-runtime.js - call: runWhatsAppInboundImageCaptionScenario - args: [{ expr: whatsappScenarioContext }] + call: runWhatsAppScenario + args: + - { expr: whatsappScenarioContext } + - { moduleExport: whatsappQaInboundImageCaptionScenario } diff --git a/qa/scenarios/channels/whatsapp-inbound-reaction-no-trigger.yaml b/qa/scenarios/channels/whatsapp-inbound-reaction-no-trigger.yaml index 9571c21c91ac..4016497ad3c4 100644 --- a/qa/scenarios/channels/whatsapp-inbound-reaction-no-trigger.yaml +++ b/qa/scenarios/channels/whatsapp-inbound-reaction-no-trigger.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 90000 retryCount: 1 suiteIsolation: isolated - config: { whatsappScenarioId: whatsapp-inbound-reaction-no-trigger } flow: module: ./live-transports/whatsapp/scenario-runtime.js - call: runWhatsAppInboundReactionNoTriggerScenario - args: [{ expr: whatsappScenarioContext }] + call: runWhatsAppScenario + args: + - { expr: whatsappScenarioContext } + - { moduleExport: whatsappQaInboundReactionNoTriggerScenario } diff --git a/qa/scenarios/channels/whatsapp-inbound-structured-messages.yaml b/qa/scenarios/channels/whatsapp-inbound-structured-messages.yaml index 0ab9545bdfa5..d1bf37508901 100644 --- a/qa/scenarios/channels/whatsapp-inbound-structured-messages.yaml +++ b/qa/scenarios/channels/whatsapp-inbound-structured-messages.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 240000 retryCount: 1 suiteIsolation: isolated - config: { whatsappScenarioId: whatsapp-inbound-structured-messages } flow: module: ./live-transports/whatsapp/scenario-runtime.js - call: runWhatsAppInboundStructuredMessagesScenario - args: [{ expr: whatsappScenarioContext }] + call: runWhatsAppScenario + args: + - { expr: whatsappScenarioContext } + - { moduleExport: whatsappQaInboundStructuredMessagesScenario } diff --git a/qa/scenarios/channels/whatsapp-mention-gating.yaml b/qa/scenarios/channels/whatsapp-mention-gating.yaml index b199b30fcefd..03db109284cc 100644 --- a/qa/scenarios/channels/whatsapp-mention-gating.yaml +++ b/qa/scenarios/channels/whatsapp-mention-gating.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 60000 retryCount: 1 suiteIsolation: isolated - config: { whatsappScenarioId: whatsapp-mention-gating } flow: module: ./live-transports/whatsapp/scenario-runtime.js - call: runWhatsAppMentionGatingScenario - args: [{ expr: whatsappScenarioContext }] + call: runWhatsAppScenario + args: + - { expr: whatsappScenarioContext } + - { moduleExport: whatsappQaMentionGatingScenario } diff --git a/qa/scenarios/channels/whatsapp-message-actions.yaml b/qa/scenarios/channels/whatsapp-message-actions.yaml index 6811660ebfa7..30461e705663 100644 --- a/qa/scenarios/channels/whatsapp-message-actions.yaml +++ b/qa/scenarios/channels/whatsapp-message-actions.yaml @@ -12,8 +12,9 @@ scenario: timeoutMs: 120000 retryCount: 1 suiteIsolation: isolated - config: { whatsappScenarioId: whatsapp-message-actions } flow: module: ./live-transports/whatsapp/scenario-runtime.js - call: runWhatsAppMessageActionsScenario - args: [{ expr: whatsappScenarioContext }] + call: runWhatsAppScenario + args: + - { expr: whatsappScenarioContext } + - { moduleExport: whatsappQaMessageActionsScenario } diff --git a/qa/scenarios/channels/whatsapp-outbound-document-preserves-filename.yaml b/qa/scenarios/channels/whatsapp-outbound-document-preserves-filename.yaml index dd4c6d6682d4..2a96576c738f 100644 --- a/qa/scenarios/channels/whatsapp-outbound-document-preserves-filename.yaml +++ b/qa/scenarios/channels/whatsapp-outbound-document-preserves-filename.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 90000 retryCount: 1 suiteIsolation: isolated - config: { whatsappScenarioId: whatsapp-outbound-document-preserves-filename } flow: module: ./live-transports/whatsapp/scenario-runtime.js - call: runWhatsAppOutboundDocumentPreservesFilenameScenario - args: [{ expr: whatsappScenarioContext }] + call: runWhatsAppScenario + args: + - { expr: whatsappScenarioContext } + - { moduleExport: whatsappQaOutboundDocumentPreservesFilenameScenario } diff --git a/qa/scenarios/channels/whatsapp-outbound-media-matrix.yaml b/qa/scenarios/channels/whatsapp-outbound-media-matrix.yaml index d435d51e4d3b..0bfef5fc60c3 100644 --- a/qa/scenarios/channels/whatsapp-outbound-media-matrix.yaml +++ b/qa/scenarios/channels/whatsapp-outbound-media-matrix.yaml @@ -13,8 +13,9 @@ scenario: timeoutMs: 120000 retryCount: 1 suiteIsolation: isolated - config: { whatsappScenarioId: whatsapp-outbound-media-matrix } flow: module: ./live-transports/whatsapp/scenario-runtime.js - call: runWhatsAppOutboundMediaMatrixScenario - args: [{ expr: whatsappScenarioContext }] + call: runWhatsAppScenario + args: + - { expr: whatsappScenarioContext } + - { moduleExport: whatsappQaOutboundMediaMatrixScenario } diff --git a/qa/scenarios/channels/whatsapp-outbound-poll.yaml b/qa/scenarios/channels/whatsapp-outbound-poll.yaml index 42978964296c..b9c270a5cad9 100644 --- a/qa/scenarios/channels/whatsapp-outbound-poll.yaml +++ b/qa/scenarios/channels/whatsapp-outbound-poll.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 90000 retryCount: 1 suiteIsolation: isolated - config: { whatsappScenarioId: whatsapp-outbound-poll } flow: module: ./live-transports/whatsapp/scenario-runtime.js - call: runWhatsAppOutboundPollScenario - args: [{ expr: whatsappScenarioContext }] + call: runWhatsAppScenario + args: + - { expr: whatsappScenarioContext } + - { moduleExport: whatsappQaOutboundPollScenario } diff --git a/qa/scenarios/channels/whatsapp-outbound-send-serialization.yaml b/qa/scenarios/channels/whatsapp-outbound-send-serialization.yaml index 99a7f006b085..c65e08b2f98f 100644 --- a/qa/scenarios/channels/whatsapp-outbound-send-serialization.yaml +++ b/qa/scenarios/channels/whatsapp-outbound-send-serialization.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 90000 retryCount: 1 suiteIsolation: isolated - config: { whatsappScenarioId: whatsapp-outbound-send-serialization } flow: module: ./live-transports/whatsapp/scenario-runtime.js - call: runWhatsAppOutboundSendSerializationScenario - args: [{ expr: whatsappScenarioContext }] + call: runWhatsAppScenario + args: + - { expr: whatsappScenarioContext } + - { moduleExport: whatsappQaOutboundSendSerializationScenario } diff --git a/qa/scenarios/channels/whatsapp-reply-context-isolation.yaml b/qa/scenarios/channels/whatsapp-reply-context-isolation.yaml index 85b7e4311948..e2210c3ff9f6 100644 --- a/qa/scenarios/channels/whatsapp-reply-context-isolation.yaml +++ b/qa/scenarios/channels/whatsapp-reply-context-isolation.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 120000 retryCount: 1 suiteIsolation: isolated - config: { whatsappScenarioId: whatsapp-reply-context-isolation } flow: module: ./live-transports/whatsapp/scenario-runtime.js - call: runWhatsAppReplyContextIsolationScenario - args: [{ expr: whatsappScenarioContext }] + call: runWhatsAppScenario + args: + - { expr: whatsappScenarioContext } + - { moduleExport: whatsappQaReplyContextIsolationScenario } diff --git a/qa/scenarios/channels/whatsapp-reply-delivery-shape.yaml b/qa/scenarios/channels/whatsapp-reply-delivery-shape.yaml index ccc7f4558850..b46bf450e78d 100644 --- a/qa/scenarios/channels/whatsapp-reply-delivery-shape.yaml +++ b/qa/scenarios/channels/whatsapp-reply-delivery-shape.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 120000 retryCount: 1 suiteIsolation: isolated - config: { whatsappScenarioId: whatsapp-reply-delivery-shape } flow: module: ./live-transports/whatsapp/scenario-runtime.js - call: runWhatsAppReplyDeliveryShapeScenario - args: [{ expr: whatsappScenarioContext }] + call: runWhatsAppScenario + args: + - { expr: whatsappScenarioContext } + - { moduleExport: whatsappQaReplyDeliveryShapeScenario } diff --git a/qa/scenarios/channels/whatsapp-reply-to-message.yaml b/qa/scenarios/channels/whatsapp-reply-to-message.yaml index 63cd1389d93b..cd7f208cacc6 100644 --- a/qa/scenarios/channels/whatsapp-reply-to-message.yaml +++ b/qa/scenarios/channels/whatsapp-reply-to-message.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 60000 retryCount: 1 suiteIsolation: isolated - config: { whatsappScenarioId: whatsapp-reply-to-message } flow: module: ./live-transports/whatsapp/scenario-runtime.js - call: runWhatsAppReplyToMessageScenario - args: [{ expr: whatsappScenarioContext }] + call: runWhatsAppScenario + args: + - { expr: whatsappScenarioContext } + - { moduleExport: whatsappQaReplyToMessageScenario } diff --git a/qa/scenarios/channels/whatsapp-reply-to-mode-batched.yaml b/qa/scenarios/channels/whatsapp-reply-to-mode-batched.yaml index 0816e433dcbd..3b2fad12670d 100644 --- a/qa/scenarios/channels/whatsapp-reply-to-mode-batched.yaml +++ b/qa/scenarios/channels/whatsapp-reply-to-mode-batched.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 90000 retryCount: 1 suiteIsolation: isolated - config: { whatsappScenarioId: whatsapp-reply-to-mode-batched } flow: module: ./live-transports/whatsapp/scenario-runtime.js - call: runWhatsAppReplyToModeBatchedScenario - args: [{ expr: whatsappScenarioContext }] + call: runWhatsAppScenario + args: + - { expr: whatsappScenarioContext } + - { moduleExport: whatsappQaReplyToModeBatchedScenario } diff --git a/qa/scenarios/channels/whatsapp-status-reaction-lifecycle.yaml b/qa/scenarios/channels/whatsapp-status-reaction-lifecycle.yaml index 1e5572bf00a0..c5d665fd6b01 100644 --- a/qa/scenarios/channels/whatsapp-status-reaction-lifecycle.yaml +++ b/qa/scenarios/channels/whatsapp-status-reaction-lifecycle.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 90000 retryCount: 1 suiteIsolation: isolated - config: { whatsappScenarioId: whatsapp-status-reaction-lifecycle } flow: module: ./live-transports/whatsapp/scenario-runtime.js - call: runWhatsAppStatusReactionLifecycleScenario - args: [{ expr: whatsappScenarioContext }] + call: runWhatsAppScenario + args: + - { expr: whatsappScenarioContext } + - { moduleExport: whatsappQaStatusReactionLifecycleScenario } diff --git a/qa/scenarios/channels/whatsapp-status-reactions.yaml b/qa/scenarios/channels/whatsapp-status-reactions.yaml index bbb6fafe545e..9092dda0a360 100644 --- a/qa/scenarios/channels/whatsapp-status-reactions.yaml +++ b/qa/scenarios/channels/whatsapp-status-reactions.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 60000 retryCount: 1 suiteIsolation: isolated - config: { whatsappScenarioId: whatsapp-status-reactions } flow: module: ./live-transports/whatsapp/scenario-runtime.js - call: runWhatsAppStatusReactionsScenario - args: [{ expr: whatsappScenarioContext }] + call: runWhatsAppScenario + args: + - { expr: whatsappScenarioContext } + - { moduleExport: whatsappQaStatusReactionsScenario } diff --git a/qa/scenarios/channels/whatsapp-stream-final-message-accounting.yaml b/qa/scenarios/channels/whatsapp-stream-final-message-accounting.yaml index 597ba05f6b28..4816eded5d2f 100644 --- a/qa/scenarios/channels/whatsapp-stream-final-message-accounting.yaml +++ b/qa/scenarios/channels/whatsapp-stream-final-message-accounting.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 90000 retryCount: 1 suiteIsolation: isolated - config: { whatsappScenarioId: whatsapp-stream-final-message-accounting } flow: module: ./live-transports/whatsapp/scenario-runtime.js - call: runWhatsAppStreamFinalMessageAccountingScenario - args: [{ expr: whatsappScenarioContext }] + call: runWhatsAppScenario + args: + - { expr: whatsappScenarioContext } + - { moduleExport: whatsappQaStreamFinalMessageAccountingScenario } diff --git a/qa/scenarios/channels/whatsapp-top-level-reply-shape.yaml b/qa/scenarios/channels/whatsapp-top-level-reply-shape.yaml index 54f2023913a7..6925981e6ce0 100644 --- a/qa/scenarios/channels/whatsapp-top-level-reply-shape.yaml +++ b/qa/scenarios/channels/whatsapp-top-level-reply-shape.yaml @@ -11,8 +11,9 @@ scenario: timeoutMs: 60000 retryCount: 1 suiteIsolation: isolated - config: { whatsappScenarioId: whatsapp-top-level-reply-shape } flow: module: ./live-transports/whatsapp/scenario-runtime.js - call: runWhatsAppTopLevelReplyShapeScenario - args: [{ expr: whatsappScenarioContext }] + call: runWhatsAppScenario + args: + - { expr: whatsappScenarioContext } + - { moduleExport: whatsappQaTopLevelReplyShapeScenario } diff --git a/src/plugin-sdk/qa-runner-runtime.ts b/src/plugin-sdk/qa-runner-runtime.ts index 9abf865015f8..fdf3136c33cd 100644 --- a/src/plugin-sdk/qa-runner-runtime.ts +++ b/src/plugin-sdk/qa-runner-runtime.ts @@ -42,6 +42,8 @@ type QaRunnerMessageRecorder = { type QaRunnerTransportFlowPreparationInput = { config: Record; + scenarioId: string; + scenarioTitle: string; gateway: { baseUrl: string; tempRoot: string;