ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-12 21:53:00 -06:00
Code Issues Packages Projects Releases Wiki Activity

refactor(qa): remove live-channel scenario registries and wrappers (#115752)

Browse Source 
* refactor(qa): remove duplicate channel scenario registries

* refactor(qa): route live scenarios through shared runners

* fix(qa): isolate WhatsApp driver retries

* refactor(qa): add module export flow arguments

* test(qa): include scenario metadata in matrix fixture
This commit is contained in:
Dallin Romney
2026-07-30 13:56:34 +08:00
committed by GitHub
parent 47a21192fd
commit 7df05d5fc0
100 changed files with 2740 additions and 2809 deletions
Download Patch File Download Diff File Expand all files Collapse all files
extensions/qa-lab/src/live-transports/discord/discord-live.runtime.test.ts
-24
View File
@@ -2,7 +2,6 @@
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
import { afterEach, describe, expect, it, vi } from "vitest";
import { discordQaScenarioSupport } from "./discord-live.runtime.js";
import { resolveDiscordQaScenarioIds } from "./scenario-selection.js";
const { testing } = discordQaScenarioSupport;
@@ -285,23 +284,6 @@ describe("discord live qa runtime", () => {
expect(testing.computeDiscordRttMs("bad", "2026-04-22T12:00:00.875Z")).toBeUndefined();
});
it("includes the Discord live scenarios", () => {
expect(testing.findScenario().map((scenario) => scenario.id)).toEqual(
resolveDiscordQaScenarioIds({}),
);
expect(
testing.findScenario(["discord-status-reactions-tool-only"]).map((scenario) => scenario.id),
).toEqual(["discord-status-reactions-tool-only"]);
expect(testing.findScenario(["discord-voice-autojoin"]).map((scenario) => scenario.id)).toEqual(
["discord-voice-autojoin"],
);
expect(
testing
.findScenario(["discord-thread-reply-filepath-attachment"])
.map((scenario) => scenario.id),
).toEqual(["discord-thread-reply-filepath-attachment"]);
});
it("collects the status reaction sequence across timeline snapshots", () => {
expect(
testing.collectSeenReactionSequence(
@@ -462,12 +444,6 @@ describe("discord live qa runtime", () => {
}
});
it("fails when any requested Discord scenario id is unknown", () => {
expect(() => testing.findScenario(["discord-canary", "typo-scenario"])).toThrow(
"unknown QA scenario id(s): typo-scenario",
);
});
it("lists Discord application commands through the REST API", async () => {
vi.stubGlobal(
"fetch",
extensions/qa-lab/src/live-transports/discord/discord-live.runtime.ts
+72 -105
View File
@@ -19,7 +19,6 @@ import { z } from "zod";
import { startQaGatewayChild } from "../../gateway-child.js";
import { isTruthyOptIn } from "../../mantis-options.runtime.js";
import { assertLiveScenarioReply as assertDiscordScenarioReply } from "../shared/live-scenario-reply.js";
import { resolveDiscordQaScenarioIds } from "./scenario-selection.js";
type DiscordQaRuntimeEnv = {
guildId: string;
@@ -30,15 +29,7 @@ type DiscordQaRuntimeEnv = {
voiceChannelId?: string;
};
type DiscordQaScenarioId =
| "discord-canary"
| "discord-mention-gating"
| "discord-native-help-command-registration"
| "discord-voice-autojoin"
| "discord-thread-reply-filepath-attachment"
| "discord-status-reactions-tool-only";
type DiscordQaScenarioRun =
export type DiscordQaScenarioRun =
| {
kind: "channel-message";
expectReply: boolean;
@@ -65,13 +56,16 @@ type DiscordQaScenarioRun =
replyContent: string;
};
type DiscordQaScenarioDefinition = {
id: DiscordQaScenarioId;
title: string;
timeoutMs: number;
export type DiscordQaScenarioImplementation = {
buildRun: (sutApplicationId: string) => DiscordQaScenarioRun;
};
type DiscordQaScenarioMetadata = {
id: string;
timeoutMs: number;
title: string;
};
type DiscordUser = {
id: string;
username?: string;
@@ -196,7 +190,7 @@ type DiscordReactionSnapshot = {
type DiscordStatusReactionTimeline = {
expectedSequence: string[];
htmlPath?: string;
scenarioId: DiscordQaScenarioId;
scenarioId: string;
scenarioTitle: string;
screenshotPath?: string;
screenshotWarning?: string;
@@ -215,7 +209,7 @@ type DiscordThreadReplyAttachmentEvidence = {
messageContent?: string;
messageId?: string;
parentMessageId?: string;
scenarioId: DiscordQaScenarioId;
scenarioId: string;
scenarioTitle: string;
screenshotPath?: string;
screenshotWarning?: string;
@@ -234,85 +228,70 @@ const DISCORD_QA_ENV_KEYS = [
"OPENCLAW_QA_DISCORD_SUT_APPLICATION_ID",
] as const;
const DISCORD_QA_SCENARIOS: DiscordQaScenarioDefinition[] = [
{
id: "discord-canary",
title: "Discord canary echo",
timeoutMs: 45_000,
buildRun: (sutApplicationId) => {
const token = `DISCORD_QA_ECHO_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
kind: "channel-message",
expectReply: true,
input: `<@${sutApplicationId}> reply with only this exact marker: ${token}`,
expectedTextIncludes: [token],
matchText: token,
};
},
export const discordQaCanaryScenario: DiscordQaScenarioImplementation = {
buildRun: (sutApplicationId) => {
const token = `DISCORD_QA_ECHO_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
kind: "channel-message",
expectReply: true,
input: `<@${sutApplicationId}> reply with only this exact marker: ${token}`,
expectedTextIncludes: [token],
matchText: token,
};
},
{
id: "discord-mention-gating",
title: "Discord unmentioned message does not trigger",
timeoutMs: 8_000,
buildRun: () => {
const token = `DISCORD_QA_NOMENTION_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
kind: "channel-message",
expectReply: false,
input: `reply with only this exact marker: ${token}`,
matchText: token,
};
},
};
export const discordQaMentionGatingScenario: DiscordQaScenarioImplementation = {
buildRun: () => {
const token = `DISCORD_QA_NOMENTION_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
kind: "channel-message",
expectReply: false,
input: `reply with only this exact marker: ${token}`,
matchText: token,
};
},
{
id: "discord-native-help-command-registration",
title: "Discord native help command is registered",
timeoutMs: 45_000,
buildRun: () => ({
kind: "application-command-registration",
expectedCommandNames: ["help"],
}),
};
export const discordQaNativeHelpCommandRegistrationScenario: DiscordQaScenarioImplementation = {
buildRun: () => ({
kind: "application-command-registration",
expectedCommandNames: ["help"],
}),
};
export const discordQaVoiceAutojoinScenario: DiscordQaScenarioImplementation = {
buildRun: () => ({
kind: "voice-autojoin",
}),
};
export const discordQaStatusReactionsToolOnlyScenario: DiscordQaScenarioImplementation = {
buildRun: () => {
const token = `DISCORD_QA_STATUS_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
kind: "status-reactions-tool-only",
input: [
`Mantis status reaction QA marker ${token}.`,
"Think briefly, then reply with only this exact marker:",
token,
].join(" "),
expectedSequence: ["👀", DEFAULT_EMOJIS.thinking, DEFAULT_EMOJIS.done],
};
},
{
id: "discord-voice-autojoin",
title: "Discord voice auto-join connects",
timeoutMs: 60_000,
buildRun: () => ({
kind: "voice-autojoin",
}),
};
export const discordQaThreadReplyFilepathAttachmentScenario: DiscordQaScenarioImplementation = {
buildRun: () => {
const token = `DISCORD_QA_THREAD_FILE_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
kind: "thread-reply-filepath-attachment",
input: `Mantis Discord thread attachment parent ${token}`,
replyContent: `Mantis thread attachment reply ${token}`,
expectedAttachmentFilename: "mantis-thread-report.md",
};
},
{
id: "discord-status-reactions-tool-only",
title: "Discord explicit status reactions run in tool-only reply mode",
timeoutMs: 75_000,
buildRun: () => {
const token = `DISCORD_QA_STATUS_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
kind: "status-reactions-tool-only",
input: [
`Mantis status reaction QA marker ${token}.`,
"Think briefly, then reply with only this exact marker:",
token,
].join(" "),
expectedSequence: ["👀", DEFAULT_EMOJIS.thinking, DEFAULT_EMOJIS.done],
};
},
},
{
id: "discord-thread-reply-filepath-attachment",
title: "Discord thread reply preserves filePath attachment",
timeoutMs: 45_000,
buildRun: () => {
const token = `DISCORD_QA_THREAD_FILE_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
kind: "thread-reply-filepath-attachment",
input: `Mantis Discord thread attachment parent ${token}`,
replyContent: `Mantis thread attachment reply ${token}`,
expectedAttachmentFilename: "mantis-thread-report.md",
};
},
},
];
};
const discordQaCredentialPayloadSchema = z.object({
guildId: z.string().trim().min(1),
@@ -951,7 +930,7 @@ async function observeStatusReactionTimeline(params: {
channelId: string;
expectedSequence: string[];
messageId: string;
scenarioId: DiscordQaScenarioId;
scenarioId: string;
scenarioTitle: string;
timeoutMs: number;
token: string;
@@ -1117,7 +1096,7 @@ async function runDiscordThreadReplyFilePathAttachmentScenario(params: {
driverBotId: string;
outputDir: string;
runtimeEnv: DiscordQaRuntimeEnv;
scenario: DiscordQaScenarioDefinition;
scenario: DiscordQaScenarioMetadata;
scenarioRun: Extract<DiscordQaScenarioRun, { kind: "thread-reply-filepath-attachment" }>;
sutAccountId: string;
sutBotId: string;
@@ -1344,16 +1323,6 @@ function buildObservedMessagesArtifact(params: {
});
}
function findScenario(ids?: string[]) {
const requestedIds = resolveDiscordQaScenarioIds({ scenarioIds: ids });
const scenariosById = new Map(DISCORD_QA_SCENARIOS.map((scenario) => [scenario.id, scenario]));
const missingIds = requestedIds.filter((id) => !scenariosById.has(id as DiscordQaScenarioId));
if (missingIds.length > 0) {
throw new Error(`unknown Discord QA scenario id(s): ${missingIds.join(", ")}`);
}
return requestedIds.map((id) => scenariosById.get(id as DiscordQaScenarioId)!);
}
function matchesDiscordScenarioReply(params: {
channelId: string;
message: DiscordObservedMessage;
@@ -1401,7 +1370,6 @@ async function assertDiscordApplicationCommandsRegistered(params: {
}
const testing = {
DISCORD_QA_SCENARIOS,
collectSeenReactionSequence,
assertDiscordScenarioReply,
assertDiscordApplicationCommandsRegistered,
@@ -1409,7 +1377,6 @@ const testing = {
buildDiscordWebMessageUrl,
buildObservedMessagesArtifact,
computeDiscordRttMs,
findScenario,
getCurrentDiscordUser,
observeStatusReactionTimeline,
pollChannelMessages,
extensions/qa-lab/src/live-transports/discord/scenario-environment.ts
+63 -53
View File
@@ -4,7 +4,11 @@ import {
patchLiveQaGatewayConfig,
readLiveQaGatewayConfig,
} from "../shared/live-gateway-config.runtime.js";
import { discordQaScenarioSupport } from "./discord-live.runtime.js";
import {
discordQaScenarioSupport,
type DiscordQaScenarioImplementation,
type DiscordQaScenarioRun,
} from "./discord-live.runtime.js";
type AdapterFactory = NonNullable<QaRunnerCliRegistration["adapterFactory"]>;
type AdapterDefinition = Awaited<ReturnType<AdapterFactory["create"]>>;
@@ -18,18 +22,21 @@ type DiscordIdentity = Awaited<
type DiscordObservedMessage = Parameters<
typeof discordQaScenarioSupport.testing.pollChannelMessages
>[0]["observedMessages"][number];
export type DiscordQaScenarioEnvironment = {
cfg: OpenClawConfig;
configureScenario: (implementation: DiscordQaScenarioImplementation) => Promise<{
cfg: OpenClawConfig;
run: DiscordQaScenarioRun;
voiceChannel?: Awaited<
ReturnType<typeof discordQaScenarioSupport.testing.resolveDiscordQaVoiceChannel>
>;
}>;
driverIdentity: DiscordIdentity;
observedMessages: DiscordObservedMessage[];
outputDir: string;
runtimeEnv: DiscordRuntimeEnv;
scenario: { id: string; timeoutMs: number; title: string };
sutAccountId: string;
sutIdentity: DiscordIdentity;
voiceChannel?: Awaited<
ReturnType<typeof discordQaScenarioSupport.testing.resolveDiscordQaVoiceChannel>
>;
};
export function createDiscordQaScenarioEnvironment(params: {
@@ -40,61 +47,64 @@ export function createDiscordQaScenarioEnvironment(params: {
}) {
const observedMessages: DiscordObservedMessage[] = [];
const prepareFlow = async (input: FlowPreparationInput) => {
const scenarioId = input.config.discordScenarioId;
if (typeof scenarioId !== "string") {
throw new Error("Discord QA module flow requires config.discordScenarioId");
}
const scenario = discordQaScenarioSupport.testing.findScenario([scenarioId])[0];
if (!scenario) {
throw new Error(`unknown Discord QA scenario id: ${scenarioId}`);
}
const scenarioRun = scenario.buildRun(params.runtimeEnv.sutApplicationId);
const voiceChannel =
scenarioRun.kind === "voice-autojoin"
? await discordQaScenarioSupport.testing.resolveDiscordQaVoiceChannel({
guildId: params.runtimeEnv.guildId,
token: params.runtimeEnv.sutBotToken,
voiceChannelId: params.runtimeEnv.voiceChannelId,
})
: undefined;
const snapshot = await readLiveQaGatewayConfig(input.gateway);
const cfg = discordQaScenarioSupport.testing.buildDiscordQaConfig(
snapshot.config as OpenClawConfig,
{
guildId: params.runtimeEnv.guildId,
channelId: params.runtimeEnv.channelId,
driverBotId: params.driverIdentity.id,
sutAccountId: params.accountId,
sutBotToken: params.runtimeEnv.sutBotToken,
},
{
...(voiceChannel
? { voiceAutoJoin: { channelId: voiceChannel.id, guildId: params.runtimeEnv.guildId } }
: {}),
statusReactionsToolOnly: scenarioRun.kind === "status-reactions-tool-only",
},
);
await patchLiveQaGatewayConfig({
gateway: input.gateway,
patch: cfg as Record<string, unknown>,
replacePaths: ["channels.discord", "messages", "plugins"],
timeoutMs: input.timeoutMs,
waitForConfigRestartSettle: input.waitForConfigRestartSettle,
});
await discordQaScenarioSupport.testing.waitForDiscordChannelRunning(
input.gateway as never,
params.accountId,
);
return {
discordScenarioContext: {
cfg,
configureScenario: async (implementation: DiscordQaScenarioImplementation) => {
const run = implementation.buildRun(params.runtimeEnv.sutApplicationId);
const voiceChannel =
run.kind === "voice-autojoin"
? await discordQaScenarioSupport.testing.resolveDiscordQaVoiceChannel({
guildId: params.runtimeEnv.guildId,
token: params.runtimeEnv.sutBotToken,
voiceChannelId: params.runtimeEnv.voiceChannelId,
})
: undefined;
const snapshot = await readLiveQaGatewayConfig(input.gateway);
const cfg = discordQaScenarioSupport.testing.buildDiscordQaConfig(
snapshot.config as OpenClawConfig,
{
guildId: params.runtimeEnv.guildId,
channelId: params.runtimeEnv.channelId,
driverBotId: params.driverIdentity.id,
sutAccountId: params.accountId,
sutBotToken: params.runtimeEnv.sutBotToken,
},
{
...(voiceChannel
? {
voiceAutoJoin: {
channelId: voiceChannel.id,
guildId: params.runtimeEnv.guildId,
},
}
: {}),
statusReactionsToolOnly: run.kind === "status-reactions-tool-only",
},
);
await patchLiveQaGatewayConfig({
gateway: input.gateway,
patch: cfg as Record<string, unknown>,
replacePaths: ["channels.discord", "messages", "plugins"],
timeoutMs: input.timeoutMs,
waitForConfigRestartSettle: input.waitForConfigRestartSettle,
});
await discordQaScenarioSupport.testing.waitForDiscordChannelRunning(
input.gateway as never,
params.accountId,
);
return { cfg, run, ...(voiceChannel ? { voiceChannel } : {}) };
},
driverIdentity: params.driverIdentity,
observedMessages,
outputDir: input.outputDir,
runtimeEnv: params.runtimeEnv,
scenario: {
id: input.scenarioId,
timeoutMs: input.timeoutMs,
title: input.scenarioTitle,
},
sutAccountId: params.accountId,
sutIdentity: params.sutIdentity,
...(voiceChannel ? { voiceChannel } : {}),
} satisfies DiscordQaScenarioEnvironment,
};
};
extensions/qa-lab/src/live-transports/discord/scenario-runtime.ts
+22 -25
View File
@@ -1,13 +1,25 @@
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
import { discordQaScenarioSupport } from "./discord-live.runtime.js";
import {
discordQaScenarioSupport,
type DiscordQaScenarioImplementation,
} from "./discord-live.runtime.js";
import type { DiscordQaScenarioEnvironment } from "./scenario-environment.js";
async function runDiscordScenario(environment: DiscordQaScenarioEnvironment, scenarioId: string) {
const scenario = discordQaScenarioSupport.testing.findScenario([scenarioId])[0];
if (!scenario) {
throw new Error(`unknown Discord QA scenario id: ${scenarioId}`);
}
const run = scenario.buildRun(environment.runtimeEnv.sutApplicationId);
export {
discordQaCanaryScenario,
discordQaMentionGatingScenario,
discordQaNativeHelpCommandRegistrationScenario,
discordQaStatusReactionsToolOnlyScenario,
discordQaThreadReplyFilepathAttachmentScenario,
discordQaVoiceAutojoinScenario,
} from "./discord-live.runtime.js";
export async function runDiscordScenario(
environment: DiscordQaScenarioEnvironment,
implementation: DiscordQaScenarioImplementation,
) {
const scenario = environment.scenario;
const { cfg, run, voiceChannel } = await environment.configureScenario(implementation);
if (run.kind === "application-command-registration") {
const registered =
await discordQaScenarioSupport.testing.assertDiscordApplicationCommandsRegistered({
@@ -19,13 +31,13 @@ async function runDiscordScenario(environment: DiscordQaScenarioEnvironment, sce
return { details: `native command registered (${registered.commandNames.join(", ")})` };
}
if (run.kind === "voice-autojoin") {
if (!environment.voiceChannel) {
if (!voiceChannel) {
throw new Error("Discord voice auto-join scenario did not resolve a voice channel.");
}
await discordQaScenarioSupport.testing.waitForDiscordVoiceState({
token: environment.runtimeEnv.sutBotToken,
guildId: environment.runtimeEnv.guildId,
channelId: environment.voiceChannel.id,
channelId: voiceChannel.id,
sutBotId: environment.sutIdentity.id,
timeoutMs: scenario.timeoutMs,
});
@@ -34,7 +46,7 @@ async function runDiscordScenario(environment: DiscordQaScenarioEnvironment, sce
if (run.kind === "thread-reply-filepath-attachment") {
const result =
await discordQaScenarioSupport.testing.runDiscordThreadReplyFilePathAttachmentScenario({
cfg: environment.cfg,
cfg,
driverBotId: environment.driverIdentity.id,
outputDir: environment.outputDir,
runtimeEnv: environment.runtimeEnv,
@@ -116,18 +128,3 @@ async function runDiscordScenario(environment: DiscordQaScenarioEnvironment, sce
throw error;
}
}
export const runDiscordCanaryScenario = (context: DiscordQaScenarioEnvironment) =>
runDiscordScenario(context, "discord-canary");
export const runDiscordMentionGatingScenario = (context: DiscordQaScenarioEnvironment) =>
runDiscordScenario(context, "discord-mention-gating");
export const runDiscordNativeHelpCommandRegistrationScenario = (
context: DiscordQaScenarioEnvironment,
) => runDiscordScenario(context, "discord-native-help-command-registration");
export const runDiscordVoiceAutojoinScenario = (context: DiscordQaScenarioEnvironment) =>
runDiscordScenario(context, "discord-voice-autojoin");
export const runDiscordStatusReactionsToolOnlyScenario = (context: DiscordQaScenarioEnvironment) =>
runDiscordScenario(context, "discord-status-reactions-tool-only");
export const runDiscordThreadReplyFilepathAttachmentScenario = (
context: DiscordQaScenarioEnvironment,
) => runDiscordScenario(context, "discord-thread-reply-filepath-attachment");
extensions/qa-lab/src/live-transports/matrix/scenarios/scenario-environment.test.ts
+6
View File
@@ -77,6 +77,8 @@ describe("matrix scenario environment", () => {
config: {},
gateway,
outputDir: "/tmp/matrix-qa/output",
scenarioId: "matrix-observer-reset",
scenarioTitle: "Matrix observer reset",
timeoutMs: 1_000,
waitForConfigRestartSettle: vi.fn(),
};
@@ -168,6 +170,8 @@ describe("matrix scenario environment", () => {
config: {},
gateway,
outputDir: "/tmp/matrix-qa/output",
scenarioId: "matrix-approval",
scenarioTitle: "Matrix approval",
timeoutMs: 1_000,
waitForConfigRestartSettle,
});
@@ -285,6 +289,8 @@ describe("matrix scenario environment", () => {
config: {},
gateway,
outputDir: "/tmp/matrix-qa/output",
scenarioId: "matrix-restart",
scenarioTitle: "Matrix restart",
timeoutMs: 1_000,
waitForConfigRestartSettle,
});
extensions/qa-lab/src/live-transports/scenario-module-parity.test.ts
+41 -15
View File
@@ -7,25 +7,36 @@ import * as whatsappScenarioRuntime from "./whatsapp/scenario-runtime.js";
const LANES = [
{
channel: "discord",
contextExpression: "discordScenarioContext",
modulePath: "./live-transports/discord/scenario-runtime.js",
runnerName: "runDiscordScenario",
runtime: discordScenarioRuntime,
},
{
channel: "slack",
contextExpression: "slackScenarioContext",
modulePath: "./live-transports/slack/scenario-runtime.js",
runnerName: "runSlackScenario",
runtime: slackScenarioRuntime,
},
{
channel: "whatsapp",
contextExpression: "whatsappScenarioContext",
modulePath: "./live-transports/whatsapp/scenario-runtime.js",
runnerName: "runWhatsAppScenario",
runtime: whatsappScenarioRuntime,
},
] as const;
function readScenarioModuleCallName(
type ScenarioModuleCall = {
args?: unknown[];
call: string;
};
function readScenarioModuleCall(
scenario: QaSeedScenarioWithSource,
modulePath: string,
): string | undefined {
): ScenarioModuleCall | undefined {
if (scenario.execution.kind !== "flow" || !scenario.execution.flow) {
return undefined;
}
@@ -48,7 +59,7 @@ function readScenarioModuleCallName(
}
const callPrefix = "scenarioModule.";
const callAction = actions.find(
(action): action is { call: string } =>
(action): action is ScenarioModuleCall =>
typeof action === "object" &&
action !== null &&
"call" in action &&
@@ -58,27 +69,42 @@ function readScenarioModuleCallName(
if (!callAction) {
throw new Error(`scenario module flow has no call: ${scenario.id}`);
}
return callAction.call.slice(callPrefix.length);
return { ...callAction, call: callAction.call.slice(callPrefix.length) };
}
describe("live transport scenario module parity", () => {
function readExpression(value: unknown): string | undefined {
return typeof value === "object" &&
value !== null &&
"expr" in value &&
typeof value.expr === "string"
? value.expr
: undefined;
}
describe("live transport scenario module routing", () => {
it.each(LANES)(
"keeps $channel scenario definitions and runtime exports in one-to-one parity",
({ channel, modulePath, runtime }) => {
"routes every $channel flow through one shared channel runner",
({ channel, contextExpression, modulePath, runnerName, runtime }) => {
expect(Reflect.get(runtime, runnerName)).toBeTypeOf("function");
const bindings = readQaScenarioPack().scenarios.flatMap((scenario) => {
if (scenario.execution.kind !== "flow" || scenario.execution.channel !== channel) {
return [];
}
const callName = readScenarioModuleCallName(scenario, modulePath);
return callName ? [{ callName, scenarioId: scenario.id }] : [];
});
const callNames = bindings.map(({ callName, scenarioId }) => {
expect(Reflect.get(runtime, callName), scenarioId).toBeTypeOf("function");
return callName;
const call = readScenarioModuleCall(scenario, modulePath);
return call ? [{ call, scenarioId: scenario.id }] : [];
});
expect(new Set(callNames).size).toBe(callNames.length);
expect(callNames.toSorted()).toEqual(Object.keys(runtime).toSorted());
for (const { call, scenarioId } of bindings) {
expect(call.call, scenarioId).toBe(runnerName);
expect(call.args, scenarioId).toHaveLength(2);
expect(readExpression(call.args?.[0]), scenarioId).toBe(contextExpression);
const implementationExpression = readExpression(call.args?.[1]);
const match = implementationExpression?.match(/^scenarioModule\["(\w+)"\]$/);
expect(match, scenarioId).toBeTruthy();
expect(Reflect.get(runtime, match?.[1] ?? ""), scenarioId).toBeTypeOf("object");
}
},
);
});
extensions/qa-lab/src/live-transports/slack/scenario-environment.test.ts
+25 -15
View File
@@ -1,5 +1,7 @@
// QA Lab Slack tests cover module-specific flow preparation boundaries.
import { describe, expect, it, vi } from "vitest";
import { createSlackQaScenarioEnvironment } from "./scenario-environment.js";
import { slackQaAllowlistBlockScenario } from "./slack-live.scenario-implementations.js";
function createEnvironment() {
return createSlackQaScenarioEnvironment({
@@ -15,21 +17,26 @@ function createEnvironment() {
});
}
describe("Slack live scenario environment", () => {
it("leaves generic declarative flows to their own config preparation", async () => {
describe("Slack scenario environment", () => {
it("leaves generic declarative flows on the adapter's baseline config", async () => {
const gatewayCall = vi.fn();
const { prepareFlow } = createEnvironment();
await expect(
prepareFlow({
config: { policyKey: "dmPolicy", policyValue: "disabled" },
gateway: { call: gatewayCall } as never,
outputDir: "/tmp/slack-output",
primaryModel: "mock-openai/gpt-5.6-luna",
timeoutMs: 60_000,
waitForConfigRestartSettle: vi.fn(),
}),
).resolves.toBeUndefined();
const prepared = await prepareFlow({
config: { replyMarker: "QA-THREAD-FOLLOW-UP-OK" },
gateway: { call: gatewayCall } as never,
outputDir: "/tmp/slack-output",
primaryModel: "mock-openai/gpt-5.6-luna",
scenarioId: "thread-follow-up",
scenarioTitle: "Thread follow-up",
timeoutMs: 60_000,
waitForConfigRestartSettle: vi.fn(),
});
expect(prepared.slackScenarioContext.scenario).toEqual({
id: "thread-follow-up",
timeoutMs: 60_000,
title: "Thread follow-up",
});
expect(gatewayCall).not.toHaveBeenCalled();
});
@@ -59,16 +66,19 @@ describe("Slack live scenario environment", () => {
throw new Error(`unexpected gateway method: ${method}`);
});
const { prepareFlow } = createEnvironment();
await prepareFlow({
config: { slackScenarioId: "slack-allowlist-block" },
const prepared = await prepareFlow({
config: {},
gateway: { call: gatewayCall } as never,
outputDir: "/tmp/slack-output",
primaryModel: "mock-openai/gpt-5.6-luna",
scenarioId: "slack-allowlist-block",
scenarioTitle: "Slack allowlist block",
timeoutMs: 60_000,
waitForConfigRestartSettle: vi.fn(),
});
await prepared.slackScenarioContext.configureScenario(slackQaAllowlistBlockScenario);
const patchCall = gatewayCall.mock.calls.find(([method]) => method === "config.patch");
if (!patchCall) {
throw new Error("config.patch was not called");
extensions/qa-lab/src/live-transports/slack/scenario-environment.ts
+45 -40
View File
@@ -10,25 +10,31 @@ import { buildSlackQaConfig } from "./slack-live.config.js";
import type {
SlackAuthIdentity,
SlackObservedMessage,
SlackQaScenarioImplementation,
SlackQaScenarioContext,
SlackQaScenarioMetadata,
SlackQaScenarioRun,
} from "./slack-live.contracts.js";
import { assertSlackCodexApprovalModelSupported } from "./slack-live.contracts.js";
import { waitForSlackChannelStable } from "./slack-live.message-observations.js";
import { sendSlackChannelMessage } from "./slack-live.observations.js";
import { getSlackQaScenarioDefinition } from "./slack-live.scenarios.js";
type AdapterFactory = NonNullable<QaRunnerCliRegistration["adapterFactory"]>;
type AdapterDefinition = Awaited<ReturnType<AdapterFactory["create"]>>;
type FlowPreparationInput = Parameters<NonNullable<AdapterDefinition["prepareFlow"]>>[0];
export type SlackQaScenarioEnvironment = {
cfg: OpenClawConfig;
channelId: string;
configureScenario: (implementation: SlackQaScenarioImplementation) => Promise<{
cfg: OpenClawConfig;
primaryModel: string;
run: SlackQaScenarioRun;
}>;
context: Omit<SlackQaScenarioContext, "sentTs">;
gatewayDebugDirPath: string;
observedMessages: SlackObservedMessage[];
outputDir: string;
primaryModel: string;
scenario: SlackQaScenarioMetadata;
stopGateway: (preserveDebugArtifacts: boolean) => Promise<void>;
sutAccountId: string;
sutIdentity: SlackAuthIdentity;
@@ -62,41 +68,6 @@ export function createSlackQaScenarioEnvironment(params: {
const observedMessages: SlackObservedMessage[] = [];
const prepareFlow = async (input: FlowPreparationInput) => {
const scenarioId = input.config.slackScenarioId;
if (typeof scenarioId !== "string") {
return undefined;
}
if (!input.primaryModel) {
throw new Error("Slack QA module flow requires a primary model");
}
const primaryModel = input.primaryModel;
const scenario = getSlackQaScenarioDefinition(scenarioId);
const scenarioRun = scenario.buildRun(params.sutIdentity.userId);
if (scenarioRun.kind === "codex-approval") {
assertSlackCodexApprovalModelSupported(primaryModel);
}
const snapshot = await readLiveQaGatewayConfig(input.gateway);
const cfg = buildSlackQaConfig(snapshot.config as OpenClawConfig, {
channelId: params.channelId,
driverBotUserId: params.driverBotUserId,
overrides: scenario.configOverrides,
primaryModel,
sutAccountId: params.accountId,
sutAppToken: params.sutAppToken,
sutBotToken: params.sutBotToken,
});
await patchLiveQaGatewayConfig({
gateway: input.gateway,
patch: cfg as Record<string, unknown>,
replacePaths: resolveSlackQaReplacePaths(params.accountId, params.channelId),
timeoutMs: input.timeoutMs,
waitForConfigRestartSettle: input.waitForConfigRestartSettle,
});
const readinessMode =
scenarioRun.kind === "approval" || scenarioRun.kind === "codex-approval"
? "started"
: "connected";
await waitForSlackChannelStable(input.gateway as never, params.accountId, readinessMode);
const context = {
channelId: params.channelId,
driverClient: params.driverClient,
@@ -115,13 +86,47 @@ export function createSlackQaScenarioEnvironment(params: {
} satisfies Omit<SlackQaScenarioContext, "sentTs">;
return {
slackScenarioContext: {
cfg,
channelId: params.channelId,
configureScenario: async (implementation: SlackQaScenarioImplementation) => {
if (!input.primaryModel) {
throw new Error("Slack QA module flow requires a primary model");
}
const primaryModel = input.primaryModel;
const run = implementation.buildRun(params.sutIdentity.userId);
if (run.kind === "codex-approval") {
assertSlackCodexApprovalModelSupported(primaryModel);
}
const snapshot = await readLiveQaGatewayConfig(input.gateway);
const cfg = buildSlackQaConfig(snapshot.config as OpenClawConfig, {
channelId: params.channelId,
driverBotUserId: params.driverBotUserId,
overrides: implementation.configOverrides,
primaryModel,
sutAccountId: params.accountId,
sutAppToken: params.sutAppToken,
sutBotToken: params.sutBotToken,
});
await patchLiveQaGatewayConfig({
gateway: input.gateway,
patch: cfg as Record<string, unknown>,
replacePaths: resolveSlackQaReplacePaths(params.accountId, params.channelId),
timeoutMs: input.timeoutMs,
waitForConfigRestartSettle: input.waitForConfigRestartSettle,
});
const readinessMode =
run.kind === "approval" || run.kind === "codex-approval" ? "started" : "connected";
await waitForSlackChannelStable(input.gateway as never, params.accountId, readinessMode);
return { cfg, primaryModel, run };
},
context,
gatewayDebugDirPath: path.join(input.outputDir, "gateway-debug"),
observedMessages,
outputDir: input.outputDir,
primaryModel,
scenario: {
id: input.scenarioId,
timeoutMs: input.timeoutMs,
title: input.scenarioTitle,
},
stopGateway: async (preserveDebugArtifacts: boolean) => {
if (!input.gateway.stop) {
throw new Error("Slack QA scenario requires gateway stop support");
extensions/qa-lab/src/live-transports/slack/scenario-runtime.ts
+33 -45
View File
@@ -1,7 +1,10 @@
import type { SlackQaScenarioEnvironment } from "./scenario-environment.js";
import { runSlackApprovalScenario } from "./slack-live.approvals.js";
import { runSlackCodexApprovalScenario } from "./slack-live.codex-approval-runner.js";
import type { SlackQaMessageScenarioRun } from "./slack-live.contracts.js";
import type {
SlackQaMessageScenarioRun,
SlackQaScenarioImplementation,
} from "./slack-live.contracts.js";
import {
observeSlackScenarioMessages,
waitForSlackNoReply,
@@ -12,7 +15,27 @@ import {
collectSlackBlockText,
sendSlackChannelMessage,
} from "./slack-live.observations.js";
import { getSlackQaScenarioDefinition } from "./slack-live.scenarios.js";
export {
slackQaAllowlistBlockScenario,
slackQaApprovalExecNativeScenario,
slackQaApprovalPluginNativeScenario,
slackQaCanaryScenario,
slackQaChannelDisabledWarningScenario,
slackQaChartPresentationNativeScenario,
slackQaCodexApprovalExecNativeScenario,
slackQaCodexApprovalPluginNativeScenario,
slackQaMentionGatingScenario,
slackQaMpimAppMentionDedupeScenario,
slackQaProgressCommentaryFalseScenario,
slackQaProgressCommentaryOmittedScenario,
slackQaProgressCommentaryTrueScenario,
slackQaProgressCommentaryVerboseDedupeScenario,
slackQaReactionGlyphNativeScenario,
slackQaTableInvalidBlocksFallbackScenario,
slackQaTablePresentationNativeScenario,
slackQaTopLevelReplyShapeScenario,
} from "./slack-live.scenario-implementations.js";
async function runSlackMessageScenario(params: {
environment: SlackQaScenarioEnvironment;
@@ -108,12 +131,15 @@ async function runSlackMessageScenario(params: {
}
}
async function runSlackScenario(environment: SlackQaScenarioEnvironment, scenarioId: string) {
const scenario = getSlackQaScenarioDefinition(scenarioId);
const run = scenario.buildRun(environment.sutIdentity.userId);
export async function runSlackScenario(
environment: SlackQaScenarioEnvironment,
implementation: SlackQaScenarioImplementation,
) {
const scenario = environment.scenario;
const { cfg, primaryModel, run } = await environment.configureScenario(implementation);
if (run.kind === "direct-transport") {
const result = await run.execute({
cfg: environment.cfg,
cfg,
channelId: environment.channelId,
sutAccountId: environment.sutAccountId,
sutIdentity: environment.sutIdentity,
@@ -159,7 +185,7 @@ async function runSlackScenario(environment: SlackQaScenarioEnvironment, scenari
channelId: environment.channelId,
context: environment.context,
observedMessages: environment.observedMessages,
primaryModel: environment.primaryModel,
primaryModel,
run,
scenario,
stopGateway: environment.stopGateway,
@@ -178,41 +204,3 @@ async function runSlackScenario(environment: SlackQaScenarioEnvironment, scenari
timeoutMs: scenario.timeoutMs,
});
}
export const runSlackCanaryScenario = (context: SlackQaScenarioEnvironment) =>
runSlackScenario(context, "slack-canary");
export const runSlackMentionGatingScenario = (context: SlackQaScenarioEnvironment) =>
runSlackScenario(context, "slack-mention-gating");
export const runSlackMpimAppMentionDedupeScenario = (context: SlackQaScenarioEnvironment) =>
runSlackScenario(context, "slack-mpim-app-mention-dedupe");
export const runSlackAllowlistBlockScenario = (context: SlackQaScenarioEnvironment) =>
runSlackScenario(context, "slack-allowlist-block");
export const runSlackChannelDisabledWarningScenario = (context: SlackQaScenarioEnvironment) =>
runSlackScenario(context, "slack-channel-disabled-warning");
export const runSlackTopLevelReplyShapeScenario = (context: SlackQaScenarioEnvironment) =>
runSlackScenario(context, "slack-top-level-reply-shape");
export const runSlackProgressCommentaryTrueScenario = (context: SlackQaScenarioEnvironment) =>
runSlackScenario(context, "slack-progress-commentary-true");
export const runSlackProgressCommentaryFalseScenario = (context: SlackQaScenarioEnvironment) =>
runSlackScenario(context, "slack-progress-commentary-false");
export const runSlackProgressCommentaryOmittedScenario = (context: SlackQaScenarioEnvironment) =>
runSlackScenario(context, "slack-progress-commentary-omitted");
export const runSlackProgressCommentaryVerboseDedupeScenario = (
context: SlackQaScenarioEnvironment,
) => runSlackScenario(context, "slack-progress-commentary-verbose-dedupe");
export const runSlackChartPresentationNativeScenario = (context: SlackQaScenarioEnvironment) =>
runSlackScenario(context, "slack-chart-presentation-native");
export const runSlackTablePresentationNativeScenario = (context: SlackQaScenarioEnvironment) =>
runSlackScenario(context, "slack-table-presentation-native");
export const runSlackTableInvalidBlocksFallbackScenario = (context: SlackQaScenarioEnvironment) =>
runSlackScenario(context, "slack-table-invalid-blocks-fallback");
export const runSlackReactionGlyphNativeScenario = (context: SlackQaScenarioEnvironment) =>
runSlackScenario(context, "slack-reaction-glyph-native");
export const runSlackApprovalExecNativeScenario = (context: SlackQaScenarioEnvironment) =>
runSlackScenario(context, "slack-approval-exec-native");
export const runSlackApprovalPluginNativeScenario = (context: SlackQaScenarioEnvironment) =>
runSlackScenario(context, "slack-approval-plugin-native");
export const runSlackCodexApprovalExecNativeScenario = (context: SlackQaScenarioEnvironment) =>
runSlackScenario(context, "slack-codex-approval-exec-native");
export const runSlackCodexApprovalPluginNativeScenario = (context: SlackQaScenarioEnvironment) =>
runSlackScenario(context, "slack-codex-approval-plugin-native");
extensions/qa-lab/src/live-transports/slack/slack-live.approval-checkpoint.ts
+1 -2
View File
@@ -9,7 +9,6 @@ import {
import {
SLACK_QA_APPROVAL_DECISION_TIMEOUT_MS,
SLACK_QA_APPROVAL_CHECKPOINT_DEFAULT_TIMEOUT_MS,
type SlackQaScenarioId,
type SlackQaApprovalKind,
type SlackQaApprovalDecision,
type SlackQaApprovalScenarioRun,
@@ -79,7 +78,7 @@ export async function writeSlackApprovalCheckpoint(params: {
decision?: SlackQaApprovalDecision;
message: SlackMessage;
observedAt: string;
scenarioId: SlackQaScenarioId;
scenarioId: string;
state: SlackApprovalCheckpointState;
}) {
const config = resolveSlackApprovalCheckpointConfig();
extensions/qa-lab/src/live-transports/slack/slack-live.approvals.ts
+2 -2
View File
@@ -13,7 +13,7 @@ import {
type SlackQaApprovalDecision,
type SlackQaApprovalScenarioRun,
type SlackQaScenarioContext,
type SlackQaScenarioDefinition,
type SlackQaScenarioMetadata,
type SlackAuthIdentity,
type SlackObservedMessage,
type SlackApprovalArtifact,
@@ -291,7 +291,7 @@ export async function runSlackApprovalScenario(params: {
context: Omit<SlackQaScenarioContext, "sentTs">;
observedMessages: SlackObservedMessage[];
run: SlackQaApprovalScenarioRun;
scenario: SlackQaScenarioDefinition;
scenario: SlackQaScenarioMetadata;
sutAccountId: string;
}) {
const requestStartedAt = new Date();
extensions/qa-lab/src/live-transports/slack/slack-live.codex-approval-runner.ts
+3 -3
View File
@@ -20,7 +20,7 @@ import {
import type {
SlackQaCodexApprovalScenarioRun,
SlackQaScenarioContext,
SlackQaScenarioDefinition,
SlackQaScenarioMetadata,
SlackObservedMessage,
SlackApprovalArtifact,
} from "./slack-live.contracts.js";
@@ -31,7 +31,7 @@ export async function runSlackCodexApprovalScenario(params: {
observedMessages: SlackObservedMessage[];
primaryModel: string;
run: SlackQaCodexApprovalScenarioRun;
scenario: SlackQaScenarioDefinition;
scenario: SlackQaScenarioMetadata;
stopGateway: (preserveDebugArtifacts: boolean) => Promise<void>;
sutAccountId: string;
}) {
@@ -97,7 +97,7 @@ async function runSlackCodexApprovalScenarioInner(params: {
observedMessages: SlackObservedMessage[];
primaryModel: string;
run: SlackQaCodexApprovalScenarioRun;
scenario: SlackQaScenarioDefinition;
scenario: SlackQaScenarioMetadata;
sutAccountId: string;
}) {
const requestStartedAt = new Date();
extensions/qa-lab/src/live-transports/slack/slack-live.codex-approval.ts
+3 -3
View File
@@ -12,7 +12,7 @@ import {
type SlackQaCodexApprovalMethod,
type SlackQaCodexApprovalScenarioRun,
type SlackQaScenarioContext,
type SlackQaScenarioDefinition,
type SlackQaScenarioMetadata,
} from "./slack-live.contracts.js";
export function resolveCodexFileApprovalTargetPath(token: string) {
@@ -224,7 +224,7 @@ export async function startCodexApprovalAgentRun(params: {
primaryModel: string;
run: SlackQaCodexApprovalScenarioRun;
runId: string;
scenario: SlackQaScenarioDefinition;
scenario: SlackQaScenarioMetadata;
sessionKey: string;
sutAccountId: string;
}) {
@@ -258,7 +258,7 @@ export async function startCodexApprovalAgentRun(params: {
}
export function buildCodexApprovalSessionKey(params: {
scenario: SlackQaScenarioDefinition;
scenario: SlackQaScenarioMetadata;
token: string;
}) {
return `agent:qa:${params.scenario.id}-${params.token.toLowerCase()}`;
extensions/qa-lab/src/live-transports/slack/slack-live.contracts.ts
+8 -27
View File
@@ -4,7 +4,6 @@ import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
import { z } from "zod";
import type { startQaGatewayChild } from "../../gateway-child.js";
import { splitQaModelRef } from "../../model-selection.js";
import type { RuntimeId } from "../../runtime-parity.js";
export type SlackQaRuntimeEnv = {
channelId: string;
@@ -85,26 +84,6 @@ export const SLACK_QA_NATIVE_TABLE = {
// These scenarios force the Codex harness, whose default provider set is intentionally narrow.
const SLACK_QA_CODEX_PROVIDER_IDS = new Set(["codex", "openai"]);
export type SlackQaScenarioId =
| "slack-allowlist-block"
| "slack-approval-exec-native"
| "slack-approval-plugin-native"
| "slack-canary"
| "slack-codex-approval-exec-native"
| "slack-codex-approval-plugin-native"
| "slack-chart-presentation-native"
| "slack-channel-disabled-warning"
| "slack-mention-gating"
| "slack-mpim-app-mention-dedupe"
| "slack-progress-commentary-false"
| "slack-progress-commentary-omitted"
| "slack-progress-commentary-true"
| "slack-progress-commentary-verbose-dedupe"
| "slack-reaction-glyph-native"
| "slack-table-invalid-blocks-fallback"
| "slack-table-presentation-native"
| "slack-top-level-reply-shape";
export type SlackQaApprovalKind = "exec" | "plugin";
export type SlackQaApprovalDecision = "allow-always" | "allow-once" | "deny";
export const SLACK_QA_APPROVAL_ACTION_PREFIX = "openclaw:approval:v1:";
@@ -184,7 +163,7 @@ export type SlackQaCodexApprovalScenarioRun = {
token: string;
};
type SlackQaScenarioRun =
export type SlackQaScenarioRun =
| SlackQaApprovalScenarioRun
| SlackQaCodexApprovalScenarioRun
| SlackQaDirectTransportScenarioRun
@@ -230,13 +209,15 @@ export type SlackQaScenarioContext = {
waitForReady: () => Promise<void>;
};
export type SlackQaScenarioDefinition = {
id: SlackQaScenarioId;
title: string;
timeoutMs: number;
export type SlackQaScenarioImplementation = {
buildRun: (sutUserId: string) => SlackQaScenarioRun;
configOverrides?: SlackQaConfigOverrides;
forcedRuntime?: RuntimeId;
};
export type SlackQaScenarioMetadata = {
id: string;
timeoutMs: number;
title: string;
};
export type SlackAuthIdentity = {
extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts
+44 -37
View File
@@ -1,5 +1,7 @@
// Qa Lab tests cover slack live plugin behavior.
import { beforeEach, describe, expect, it, vi } from "vitest";
import { readQaScenarioById } from "../../scenario-catalog.js";
import { requireFlowScenario } from "../../scenario-catalog.test-utils.js";
import { testing as adapterTesting } from "./adapter.runtime.js";
import { resolveSlackQaScenarioIds } from "./scenario-selection.js";
import { resolveApprovalDecision } from "./slack-live.approvals.js";
@@ -13,7 +15,10 @@ import {
parseSlackQaCredentialPayload,
resolveSlackQaRuntimeEnv,
} from "./slack-live.config.js";
import { assertSlackCodexApprovalModelSupported } from "./slack-live.contracts.js";
import {
assertSlackCodexApprovalModelSupported,
type SlackQaScenarioImplementation,
} from "./slack-live.contracts.js";
import { buildSlackInvalidBlocksTableProbe } from "./slack-live.invalid-blocks.js";
import {
observeSlackScenarioMessages,
@@ -25,16 +30,32 @@ import {
extractSlackNativeApprovalId,
runSlackTableInvalidBlocksFallbackScenario,
} from "./slack-live.observations.js";
import {
getSlackQaScenarioDefinition,
listSlackQaScenarioCatalog,
} from "./slack-live.scenarios.js";
import * as slackScenarioImplementations from "./slack-live.scenario-implementations.js";
function toSlackScenarioExportName(id: string): string {
const suffix = id
.replace(/^slack-/, "")
.split("-")
.map((part) => `${part.slice(0, 1).toUpperCase()}${part.slice(1)}`)
.join("");
return `slackQa${suffix}Scenario`;
}
function findScenario(ids?: string[]) {
const requestedIds = new Set(ids?.length ? ids : resolveSlackQaScenarioIds({}));
return listSlackQaScenarioCatalog()
.filter(({ id }) => requestedIds.has(id))
.map(({ id }) => getSlackQaScenarioDefinition(id));
return resolveSlackQaScenarioIds({ scenarioIds: ids }).map((id) => {
const implementation = (
slackScenarioImplementations as unknown as Record<string, SlackQaScenarioImplementation>
)[toSlackScenarioExportName(id)];
if (!implementation) {
throw new Error(`missing Slack test implementation for ${id}`);
}
const scenario = requireFlowScenario(readQaScenarioById(id));
return Object.assign({}, implementation, {
id,
timeoutMs: scenario.execution.timeoutMs ?? 60_000,
title: scenario.title,
});
});
}
const testing = {
@@ -207,43 +228,29 @@ describe("Slack live QA runtime helpers", () => {
});
it("selects native scenarios by explicit id", () => {
expect(
testing
.findScenario([
"slack-chart-presentation-native",
"slack-table-presentation-native",
"slack-table-invalid-blocks-fallback",
"slack-progress-commentary-true",
"slack-progress-commentary-false",
"slack-progress-commentary-omitted",
"slack-progress-commentary-verbose-dedupe",
"slack-reaction-glyph-native",
"slack-approval-exec-native",
"slack-approval-plugin-native",
"slack-codex-approval-exec-native",
"slack-codex-approval-plugin-native",
"slack-channel-disabled-warning",
])
.map((scenario) => scenario.id),
).toEqual([
"slack-channel-disabled-warning",
const scenarioIds = [
"slack-chart-presentation-native",
"slack-table-presentation-native",
"slack-table-invalid-blocks-fallback",
"slack-progress-commentary-true",
"slack-progress-commentary-false",
"slack-progress-commentary-omitted",
"slack-progress-commentary-verbose-dedupe",
"slack-chart-presentation-native",
"slack-table-presentation-native",
"slack-table-invalid-blocks-fallback",
"slack-reaction-glyph-native",
"slack-approval-exec-native",
"slack-approval-plugin-native",
"slack-codex-approval-exec-native",
"slack-codex-approval-plugin-native",
]);
expect(testing.findScenario(["slack-codex-approval-exec-native"])[0]?.forcedRuntime).toBe(
"codex",
);
expect(testing.findScenario(["slack-canary"])[0]?.forcedRuntime).toBeUndefined();
"slack-channel-disabled-warning",
];
const selectedIds = testing.findScenario(scenarioIds).map((scenario) => scenario.id);
expect(new Set(selectedIds)).toEqual(new Set(scenarioIds));
expect(
requireFlowScenario(readQaScenarioById("slack-codex-approval-exec-native")).execution.runtime,
).toBe("codex");
expect(
requireFlowScenario(readQaScenarioById("slack-canary")).execution.runtime,
).toBeUndefined();
});
it("accepts only Codex harness providers for Codex approval scenarios", () => {
extensions/qa-lab/src/live-transports/slack/slack-live.scenario-implementations.ts
+417
View File
@@ -0,0 +1,417 @@
// QA Lab Slack live scenario implementations.
import { randomUUID } from "node:crypto";
import { setTimeout as sleep } from "node:timers/promises";
import { waitForSlackReaction } from "./slack-live.codex-approval.js";
import {
SLACK_QA_REACTION_VERIFY_TIMEOUT_MS,
SLACK_QA_NATIVE_DATA_VERIFY_TIMEOUT_MS,
SLACK_QA_LOG_TAIL_TIMEOUT_MS,
type SlackQaScenarioImplementation,
type SlackQaScenarioContext,
} from "./slack-live.contracts.js";
import {
isExpectedSlackNativeChartMessage,
isExpectedSlackNativeTableMessage,
runSlackTableInvalidBlocksFallbackScenario,
waitForSlackStoredMessage,
} from "./slack-live.observations.js";
import {
buildSlackChartMessageToolArgs,
renderSlackChartAccessibleText,
buildSlackTableMessageToolArgs,
renderSlackTableAccessibleText,
buildSlackProgressCommentaryRun,
} from "./slack-live.scenario-fixtures.js";
export const slackQaCanaryScenario: SlackQaScenarioImplementation = {
buildRun: (sutUserId) => {
const token = `SLACK_QA_ECHO_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
expectReply: true,
input: `<@${sutUserId}> reply with only this exact marker: ${token}`,
matchText: token,
};
},
};
export const slackQaMentionGatingScenario: SlackQaScenarioImplementation = {
buildRun: () => {
const token = `SLACK_QA_NOMENTION_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
expectReply: false,
input: `reply with only this exact marker: ${token}`,
matchText: token,
};
},
};
export const slackQaMpimAppMentionDedupeScenario: SlackQaScenarioImplementation = {
configOverrides: { groupDmEnabled: true },
buildRun: (sutUserId) => {
const token = `SLACK_QA_MPIM_${randomUUID().slice(0, 8).toUpperCase()}`;
let openedChannelId: string | undefined;
const closeOpenedChannel = async (context: Omit<SlackQaScenarioContext, "sentTs">) => {
if (!openedChannelId) {
return;
}
const channelId = openedChannelId;
for (let attempt = 1; attempt <= 2; attempt += 1) {
try {
await context.sutReadClient.conversations.close({ channel: channelId });
openedChannelId = undefined;
return;
} catch (error) {
if (attempt === 2) {
throw error;
}
// Retain ownership until Slack confirms closure; one bounded retry
// covers a transient API failure without hiding a leaked MPIM.
await sleep(500);
}
}
};
return {
expectReply: true,
input: `<@${sutUserId}> reply with only this exact marker: ${token}`,
matchText: token,
settleObservedMs: 60_000,
beforeRun: async (context) => {
const driverAuth = await context.driverClient.auth.test();
const driverUserId = driverAuth.user_id?.trim();
if (!driverUserId) {
throw new Error("Slack QA driver auth.test returned no user_id");
}
const members = await context.sutReadClient.conversations.members({
channel: context.channelId,
limit: 100,
});
const candidateUserIds = (members.members ?? []).filter(
(userId) => userId !== driverUserId && userId !== context.sutIdentity.userId,
);
for (const userId of candidateUserIds) {
const user = (await context.sutReadClient.users.info({ user: userId })).user;
if (!user || user.deleted || user.is_bot) {
continue;
}
const opened = await context.sutReadClient.conversations.open({
return_im: true,
users: `${driverUserId},${userId}`,
});
const channelId = opened.channel?.id?.trim();
if (!channelId) {
continue;
}
// Track ownership before the metadata call so outer cleanup can still
// close the MPIM when Slack rejects or times out during inspection.
openedChannelId = channelId;
const info = await context.sutReadClient.conversations.info({ channel: channelId });
if (info.channel?.is_mpim && channelId.startsWith("C")) {
return { details: "opened C-prefixed MPIM", inputChannelId: channelId };
}
await closeOpenedChannel(context);
}
throw new Error("Slack QA channel has no human member yielding a C-prefixed MPIM");
},
verifyObserved: ({ messages }) => {
const uniqueReplies = new Map(messages.map((message) => [message.ts, message]));
const matchingReplies = [...uniqueReplies.values()].filter((message) =>
message.text.includes(token),
);
if (uniqueReplies.size !== 1 || matchingReplies.length !== 1) {
throw new Error(
`expected one MPIM response with the marker, got ${uniqueReplies.size} response(s) and ${matchingReplies.length} marker match(es)`,
);
}
return "one MPIM reply observed after message/app_mention twin delivery";
},
cleanup: closeOpenedChannel,
};
},
};
export const slackQaAllowlistBlockScenario: SlackQaScenarioImplementation = {
configOverrides: {
allowFrom: ["U_OPENCLAW_QA_NEVER_ALLOWED"],
users: ["U_OPENCLAW_QA_NEVER_ALLOWED"],
},
buildRun: (sutUserId) => {
const token = `SLACK_QA_BLOCK_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
expectReply: false,
input: `<@${sutUserId}> reply with only this exact marker: ${token}`,
matchText: token,
};
},
};
export const slackQaChannelDisabledWarningScenario: SlackQaScenarioImplementation = {
configOverrides: { channelEnabled: false },
buildRun: (sutUserId) => {
const marker = `SLACK_QA_DISABLED_${randomUUID().slice(0, 8).toUpperCase()}`;
let logCursor = 0;
return {
expectReply: false,
input: `<@${sutUserId}> reply with only this exact marker: ${marker}`,
matchText: marker,
preserveGatewayDebug: true,
beforeRun: async ({ gateway }) => {
const gatewayLogTail = (await gateway.call(
"logs.tail",
{ limit: 1, maxBytes: 32_000 },
{ timeoutMs: SLACK_QA_LOG_TAIL_TIMEOUT_MS },
)) as { cursor?: unknown };
logCursor = typeof gatewayLogTail.cursor === "number" ? gatewayLogTail.cursor : 0;
},
afterNoReply: async ({ gateway }) => {
const gatewayLogTail = (await gateway.call(
"logs.tail",
{ cursor: logCursor, limit: 200, maxBytes: 256_000 },
{ timeoutMs: SLACK_QA_LOG_TAIL_TIMEOUT_MS },
)) as { lines?: unknown };
const gatewayLogLines = Array.isArray(gatewayLogTail.lines)
? gatewayLogTail.lines.filter((line): line is string => typeof line === "string")
: [];
const expectedFields = [
"Slack channel denied by configuration",
"channel_not_allowed",
"channel_disabled",
];
if (
!gatewayLogLines.some((line) => expectedFields.every((field) => line.includes(field)))
) {
throw new Error("disabled Slack channel did not emit the structured warning");
}
return "structured disabled-channel warning observed";
},
};
},
};
export const slackQaTopLevelReplyShapeScenario: SlackQaScenarioImplementation = {
configOverrides: { replyToMode: "off" },
buildRun: (sutUserId) => {
const token = `SLACK_QA_TOPLEVEL_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
expectReply: true,
input: `<@${sutUserId}> reply with only this exact marker: ${token}`,
matchText: token,
verify: (message) => {
if (message.thread_ts) {
throw new Error(
`expected top-level Slack reply without thread_ts; got ${message.thread_ts}`,
);
}
},
};
},
};
export const slackQaProgressCommentaryTrueScenario: SlackQaScenarioImplementation = {
configOverrides: {
progress: { commentary: true, toolProgress: false },
},
buildRun: (sutUserId) =>
buildSlackProgressCommentaryRun(sutUserId, {
commentary: "draft",
toolProgress: "absent",
}),
};
export const slackQaProgressCommentaryFalseScenario: SlackQaScenarioImplementation = {
configOverrides: {
progress: { commentary: false, toolProgress: false },
},
buildRun: (sutUserId) =>
buildSlackProgressCommentaryRun(sutUserId, {
commentary: "absent",
toolProgress: "absent",
}),
};
export const slackQaProgressCommentaryOmittedScenario: SlackQaScenarioImplementation = {
configOverrides: {
progress: { toolProgress: true },
},
buildRun: (sutUserId) =>
buildSlackProgressCommentaryRun(sutUserId, {
commentary: "draft",
toolProgress: "draft",
}),
};
export const slackQaProgressCommentaryVerboseDedupeScenario: SlackQaScenarioImplementation = {
configOverrides: {
progress: { commentary: true, toolProgress: false, verboseDefault: "on" },
},
buildRun: (sutUserId) =>
buildSlackProgressCommentaryRun(sutUserId, {
commentary: "standalone",
toolProgress: "standalone",
}),
};
export const slackQaChartPresentationNativeScenario: SlackQaScenarioImplementation = {
configOverrides: { messageTool: true },
buildRun: (sutUserId) => {
const suffix = randomUUID().slice(0, 8).toUpperCase();
const summaryText = `SLACK_QA_CHART_SUMMARY_${suffix}`;
const finalMarker = `SLACK_QA_CHART_DONE_${suffix}`;
const messageToolArgs = buildSlackChartMessageToolArgs(summaryText);
return {
expectReply: true,
input: [
`<@${sutUserId}> Slack native chart QA check ${summaryText}.`,
`Call the message tool exactly once with these exact arguments: ${JSON.stringify(messageToolArgs)}.`,
`After the chart send succeeds, reply with only this exact marker: ${finalMarker}`,
].join(" "),
matchText: finalMarker,
afterReply: async (_message, context) => {
await waitForSlackStoredMessage({
channelId: context.channelId,
client: context.sutReadClient,
description: "message with native chart",
matchesMessage: (message) =>
isExpectedSlackNativeChartMessage(message, renderSlackChartAccessibleText(summaryText)),
oldestTs: context.sentTs,
sutIdentity: context.sutIdentity,
timeoutMs: SLACK_QA_NATIVE_DATA_VERIFY_TIMEOUT_MS,
});
return "verified native data_visualization block and deterministic accessible text";
},
};
},
};
export const slackQaTablePresentationNativeScenario: SlackQaScenarioImplementation = {
configOverrides: { messageTool: true },
buildRun: (sutUserId) => {
const suffix = randomUUID().slice(0, 8).toUpperCase();
const summaryText = `SLACK_QA_TABLE_SUMMARY_${suffix}`;
const finalMarker = `SLACK_QA_TABLE_DONE_${suffix}`;
const messageToolArgs = buildSlackTableMessageToolArgs(summaryText);
return {
expectReply: true,
input: [
`<@${sutUserId}> Slack native table QA check ${summaryText}.`,
`Call the message tool exactly once with these exact arguments: ${JSON.stringify(messageToolArgs)}.`,
`After the table send succeeds, reply with only this exact marker: ${finalMarker}`,
].join(" "),
matchText: finalMarker,
afterReply: async (_message, context) => {
await waitForSlackStoredMessage({
channelId: context.channelId,
client: context.sutReadClient,
description: "message with native table",
matchesMessage: (message) =>
isExpectedSlackNativeTableMessage(message, renderSlackTableAccessibleText(summaryText)),
oldestTs: context.sentTs,
sutIdentity: context.sutIdentity,
timeoutMs: SLACK_QA_NATIVE_DATA_VERIFY_TIMEOUT_MS,
});
return "verified native data_table block and deterministic accessible text";
},
};
},
};
export const slackQaTableInvalidBlocksFallbackScenario: SlackQaScenarioImplementation = {
buildRun: () => ({
kind: "direct-transport",
execute: runSlackTableInvalidBlocksFallbackScenario,
}),
};
export const slackQaReactionGlyphNativeScenario: SlackQaScenarioImplementation = {
configOverrides: { messageTool: true },
buildRun: (sutUserId) => {
const token = `SLACK_QA_REACTION_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
expectReply: true,
input: [
`<@${sutUserId}> use the message tool exactly once to react to this message.`,
'Set action to "react", channel to "slack", and emoji to exactly "✅".',
"Do not substitute a shortcode.",
`After the reaction succeeds, reply with only this exact marker: ${token}`,
].join(" "),
matchText: token,
afterReply: async (_message, context) => {
await waitForSlackReaction({
channelId: context.channelId,
client: context.sutReadClient,
expectedReactionName: "white_check_mark",
messageId: context.sentTs,
sutUserId: context.sutIdentity.userId,
timeoutMs: SLACK_QA_REACTION_VERIFY_TIMEOUT_MS,
});
return "verified SUT white_check_mark reaction from exact glyph instruction";
},
};
},
};
export const slackQaApprovalExecNativeScenario: SlackQaScenarioImplementation = {
configOverrides: {
approvals: {
exec: true,
target: "channel",
},
},
buildRun: () => ({
approvalKind: "exec",
decision: "allow-once",
kind: "approval",
token: `SLACK_QA_EXEC_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`,
}),
};
export const slackQaApprovalPluginNativeScenario: SlackQaScenarioImplementation = {
configOverrides: {
approvals: {
exec: true,
plugin: true,
target: "channel",
},
},
buildRun: () => ({
approvalKind: "plugin",
decision: "allow-once",
kind: "approval",
token: `SLACK_QA_PLUGIN_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`,
}),
};
export const slackQaCodexApprovalExecNativeScenario: SlackQaScenarioImplementation = {
configOverrides: {
approvals: {
exec: true,
plugin: true,
target: "channel",
},
codexApproval: true,
},
buildRun: () => ({
approvalKind: "plugin",
appServerMethod: "item/commandExecution/requestApproval",
decision: "allow-once",
kind: "codex-approval",
token: `SLACK_QA_CODEX_EXEC_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`,
}),
};
export const slackQaCodexApprovalPluginNativeScenario: SlackQaScenarioImplementation = {
configOverrides: {
approvals: {
exec: true,
plugin: true,
target: "channel",
},
codexApproval: true,
},
buildRun: () => ({
approvalKind: "plugin",
appServerMethod: "item/fileChange/requestApproval",
decision: "allow-once",
kind: "codex-approval",
token: `SLACK_QA_CODEX_FILE_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`,
}),
};
extensions/qa-lab/src/live-transports/slack/slack-live.scenarios.ts
-476
View File
@@ -1,476 +0,0 @@
// QA Lab Slack live scenario catalog.
import { randomUUID } from "node:crypto";
import { setTimeout as sleep } from "node:timers/promises";
import { waitForSlackReaction } from "./slack-live.codex-approval.js";
import {
SLACK_QA_REACTION_VERIFY_TIMEOUT_MS,
SLACK_QA_NATIVE_DATA_VERIFY_TIMEOUT_MS,
SLACK_QA_LOG_TAIL_TIMEOUT_MS,
type SlackQaScenarioDefinition,
type SlackQaScenarioContext,
} from "./slack-live.contracts.js";
import {
isExpectedSlackNativeChartMessage,
isExpectedSlackNativeTableMessage,
runSlackTableInvalidBlocksFallbackScenario,
waitForSlackStoredMessage,
} from "./slack-live.observations.js";
import {
buildSlackChartMessageToolArgs,
renderSlackChartAccessibleText,
buildSlackTableMessageToolArgs,
renderSlackTableAccessibleText,
buildSlackProgressCommentaryRun,
} from "./slack-live.scenario-fixtures.js";
const SLACK_QA_SCENARIOS: SlackQaScenarioDefinition[] = [
{
id: "slack-canary",
title: "Slack canary echo",
timeoutMs: 45_000,
buildRun: (sutUserId) => {
const token = `SLACK_QA_ECHO_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
expectReply: true,
input: `<@${sutUserId}> reply with only this exact marker: ${token}`,
matchText: token,
};
},
},
{
id: "slack-mention-gating",
title: "Slack unmentioned bot message does not trigger",
timeoutMs: 8_000,
buildRun: () => {
const token = `SLACK_QA_NOMENTION_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
expectReply: false,
input: `reply with only this exact marker: ${token}`,
matchText: token,
};
},
},
{
id: "slack-mpim-app-mention-dedupe",
title: "Slack MPIM app mention dispatches once",
timeoutMs: 90_000,
configOverrides: { groupDmEnabled: true },
buildRun: (sutUserId) => {
const token = `SLACK_QA_MPIM_${randomUUID().slice(0, 8).toUpperCase()}`;
let openedChannelId: string | undefined;
const closeOpenedChannel = async (context: Omit<SlackQaScenarioContext, "sentTs">) => {
if (!openedChannelId) {
return;
}
const channelId = openedChannelId;
for (let attempt = 1; attempt <= 2; attempt += 1) {
try {
await context.sutReadClient.conversations.close({ channel: channelId });
openedChannelId = undefined;
return;
} catch (error) {
if (attempt === 2) {
throw error;
}
// Retain ownership until Slack confirms closure; one bounded retry
// covers a transient API failure without hiding a leaked MPIM.
await sleep(500);
}
}
};
return {
expectReply: true,
input: `<@${sutUserId}> reply with only this exact marker: ${token}`,
matchText: token,
settleObservedMs: 60_000,
beforeRun: async (context) => {
const driverAuth = await context.driverClient.auth.test();
const driverUserId = driverAuth.user_id?.trim();
if (!driverUserId) {
throw new Error("Slack QA driver auth.test returned no user_id");
}
const members = await context.sutReadClient.conversations.members({
channel: context.channelId,
limit: 100,
});
const candidateUserIds = (members.members ?? []).filter(
(userId) => userId !== driverUserId && userId !== context.sutIdentity.userId,
);
for (const userId of candidateUserIds) {
const user = (await context.sutReadClient.users.info({ user: userId })).user;
if (!user || user.deleted || user.is_bot) {
continue;
}
const opened = await context.sutReadClient.conversations.open({
return_im: true,
users: `${driverUserId},${userId}`,
});
const channelId = opened.channel?.id?.trim();
if (!channelId) {
continue;
}
// Track ownership before the metadata call so outer cleanup can still
// close the MPIM when Slack rejects or times out during inspection.
openedChannelId = channelId;
const info = await context.sutReadClient.conversations.info({ channel: channelId });
if (info.channel?.is_mpim && channelId.startsWith("C")) {
return { details: "opened C-prefixed MPIM", inputChannelId: channelId };
}
await closeOpenedChannel(context);
}
throw new Error("Slack QA channel has no human member yielding a C-prefixed MPIM");
},
verifyObserved: ({ messages }) => {
const uniqueReplies = new Map(messages.map((message) => [message.ts, message]));
const matchingReplies = [...uniqueReplies.values()].filter((message) =>
message.text.includes(token),
);
if (uniqueReplies.size !== 1 || matchingReplies.length !== 1) {
throw new Error(
`expected one MPIM response with the marker, got ${uniqueReplies.size} response(s) and ${matchingReplies.length} marker match(es)`,
);
}
return "one MPIM reply observed after message/app_mention twin delivery";
},
cleanup: closeOpenedChannel,
};
},
},
{
id: "slack-allowlist-block",
title: "Slack non-allowlisted sender does not trigger",
timeoutMs: 8_000,
configOverrides: {
allowFrom: ["U_OPENCLAW_QA_NEVER_ALLOWED"],
users: ["U_OPENCLAW_QA_NEVER_ALLOWED"],
},
buildRun: (sutUserId) => {
const token = `SLACK_QA_BLOCK_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
expectReply: false,
input: `<@${sutUserId}> reply with only this exact marker: ${token}`,
matchText: token,
};
},
},
{
id: "slack-channel-disabled-warning",
title: "Slack disabled channel warns and does not trigger",
timeoutMs: 8_000,
configOverrides: { channelEnabled: false },
buildRun: (sutUserId) => {
const marker = `SLACK_QA_DISABLED_${randomUUID().slice(0, 8).toUpperCase()}`;
let logCursor = 0;
return {
expectReply: false,
input: `<@${sutUserId}> reply with only this exact marker: ${marker}`,
matchText: marker,
preserveGatewayDebug: true,
beforeRun: async ({ gateway }) => {
const gatewayLogTail = (await gateway.call(
"logs.tail",
{ limit: 1, maxBytes: 32_000 },
{ timeoutMs: SLACK_QA_LOG_TAIL_TIMEOUT_MS },
)) as { cursor?: unknown };
logCursor = typeof gatewayLogTail.cursor === "number" ? gatewayLogTail.cursor : 0;
},
afterNoReply: async ({ gateway }) => {
const gatewayLogTail = (await gateway.call(
"logs.tail",
{ cursor: logCursor, limit: 200, maxBytes: 256_000 },
{ timeoutMs: SLACK_QA_LOG_TAIL_TIMEOUT_MS },
)) as { lines?: unknown };
const gatewayLogLines = Array.isArray(gatewayLogTail.lines)
? gatewayLogTail.lines.filter((line): line is string => typeof line === "string")
: [];
const expectedFields = [
"Slack channel denied by configuration",
"channel_not_allowed",
"channel_disabled",
];
if (
!gatewayLogLines.some((line) => expectedFields.every((field) => line.includes(field)))
) {
throw new Error("disabled Slack channel did not emit the structured warning");
}
return "structured disabled-channel warning observed";
},
};
},
},
{
id: "slack-top-level-reply-shape",
title: "Slack top-level reply stays top-level",
timeoutMs: 45_000,
configOverrides: { replyToMode: "off" },
buildRun: (sutUserId) => {
const token = `SLACK_QA_TOPLEVEL_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
expectReply: true,
input: `<@${sutUserId}> reply with only this exact marker: ${token}`,
matchText: token,
verify: (message) => {
if (message.thread_ts) {
throw new Error(
`expected top-level Slack reply without thread_ts; got ${message.thread_ts}`,
);
}
},
};
},
},
{
id: "slack-progress-commentary-true",
title: "Slack progress commentary true is independent from tool progress",
timeoutMs: 90_000,
configOverrides: {
progress: { commentary: true, toolProgress: false },
},
buildRun: (sutUserId) =>
buildSlackProgressCommentaryRun(sutUserId, {
commentary: "draft",
toolProgress: "absent",
}),
},
{
id: "slack-progress-commentary-false",
title: "Slack progress commentary false stays out of the progress draft",
timeoutMs: 90_000,
configOverrides: {
progress: { commentary: false, toolProgress: false },
},
buildRun: (sutUserId) =>
buildSlackProgressCommentaryRun(sutUserId, {
commentary: "absent",
toolProgress: "absent",
}),
},
{
id: "slack-progress-commentary-omitted",
title: "Slack omitted progress commentary preserves the tool-progress default",
timeoutMs: 90_000,
configOverrides: {
progress: { toolProgress: true },
},
buildRun: (sutUserId) =>
buildSlackProgressCommentaryRun(sutUserId, {
commentary: "draft",
toolProgress: "draft",
}),
},
{
id: "slack-progress-commentary-verbose-dedupe",
title: "Slack explicit commentary yields to durable verbose progress",
timeoutMs: 90_000,
configOverrides: {
progress: { commentary: true, toolProgress: false, verboseDefault: "on" },
},
buildRun: (sutUserId) =>
buildSlackProgressCommentaryRun(sutUserId, {
commentary: "standalone",
toolProgress: "standalone",
}),
},
{
id: "slack-chart-presentation-native",
title: "Slack portable chart renders as a native data visualization",
timeoutMs: 90_000,
configOverrides: { messageTool: true },
buildRun: (sutUserId) => {
const suffix = randomUUID().slice(0, 8).toUpperCase();
const summaryText = `SLACK_QA_CHART_SUMMARY_${suffix}`;
const finalMarker = `SLACK_QA_CHART_DONE_${suffix}`;
const messageToolArgs = buildSlackChartMessageToolArgs(summaryText);
return {
expectReply: true,
input: [
`<@${sutUserId}> Slack native chart QA check ${summaryText}.`,
`Call the message tool exactly once with these exact arguments: ${JSON.stringify(messageToolArgs)}.`,
`After the chart send succeeds, reply with only this exact marker: ${finalMarker}`,
].join(" "),
matchText: finalMarker,
afterReply: async (_message, context) => {
await waitForSlackStoredMessage({
channelId: context.channelId,
client: context.sutReadClient,
description: "message with native chart",
matchesMessage: (message) =>
isExpectedSlackNativeChartMessage(
message,
renderSlackChartAccessibleText(summaryText),
),
oldestTs: context.sentTs,
sutIdentity: context.sutIdentity,
timeoutMs: SLACK_QA_NATIVE_DATA_VERIFY_TIMEOUT_MS,
});
return "verified native data_visualization block and deterministic accessible text";
},
};
},
},
{
id: "slack-table-presentation-native",
title: "Slack portable table renders as a native data table",
timeoutMs: 90_000,
configOverrides: { messageTool: true },
buildRun: (sutUserId) => {
const suffix = randomUUID().slice(0, 8).toUpperCase();
const summaryText = `SLACK_QA_TABLE_SUMMARY_${suffix}`;
const finalMarker = `SLACK_QA_TABLE_DONE_${suffix}`;
const messageToolArgs = buildSlackTableMessageToolArgs(summaryText);
return {
expectReply: true,
input: [
`<@${sutUserId}> Slack native table QA check ${summaryText}.`,
`Call the message tool exactly once with these exact arguments: ${JSON.stringify(messageToolArgs)}.`,
`After the table send succeeds, reply with only this exact marker: ${finalMarker}`,
].join(" "),
matchText: finalMarker,
afterReply: async (_message, context) => {
await waitForSlackStoredMessage({
channelId: context.channelId,
client: context.sutReadClient,
description: "message with native table",
matchesMessage: (message) =>
isExpectedSlackNativeTableMessage(
message,
renderSlackTableAccessibleText(summaryText),
),
oldestTs: context.sentTs,
sutIdentity: context.sutIdentity,
timeoutMs: SLACK_QA_NATIVE_DATA_VERIFY_TIMEOUT_MS,
});
return "verified native data_table block and deterministic accessible text";
},
};
},
},
{
id: "slack-table-invalid-blocks-fallback",
title: "Slack rejects an over-limit native table and stores its complete fallback",
timeoutMs: 45_000,
buildRun: () => ({
kind: "direct-transport",
execute: runSlackTableInvalidBlocksFallbackScenario,
}),
},
{
id: "slack-reaction-glyph-native",
title: "Slack message tool normalizes an emoji glyph reaction",
timeoutMs: 90_000,
configOverrides: { messageTool: true },
buildRun: (sutUserId) => {
const token = `SLACK_QA_REACTION_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
expectReply: true,
input: [
`<@${sutUserId}> use the message tool exactly once to react to this message.`,
'Set action to "react", channel to "slack", and emoji to exactly "✅".',
"Do not substitute a shortcode.",
`After the reaction succeeds, reply with only this exact marker: ${token}`,
].join(" "),
matchText: token,
afterReply: async (_message, context) => {
await waitForSlackReaction({
channelId: context.channelId,
client: context.sutReadClient,
expectedReactionName: "white_check_mark",
messageId: context.sentTs,
sutUserId: context.sutIdentity.userId,
timeoutMs: SLACK_QA_REACTION_VERIFY_TIMEOUT_MS,
});
return "verified SUT white_check_mark reaction from exact glyph instruction";
},
};
},
},
{
id: "slack-approval-exec-native",
title: "Slack native exec approval prompt resolves",
timeoutMs: 60_000,
configOverrides: {
approvals: {
exec: true,
target: "channel",
},
},
buildRun: () => ({
approvalKind: "exec",
decision: "allow-once",
kind: "approval",
token: `SLACK_QA_EXEC_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`,
}),
},
{
id: "slack-approval-plugin-native",
title: "Slack native plugin approval prompt resolves with exec approvals enabled",
timeoutMs: 60_000,
configOverrides: {
approvals: {
exec: true,
plugin: true,
target: "channel",
},
},
buildRun: () => ({
approvalKind: "plugin",
decision: "allow-once",
kind: "approval",
token: `SLACK_QA_PLUGIN_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`,
}),
},
{
id: "slack-codex-approval-exec-native",
title: "Slack native Codex command approval prompt resolves",
timeoutMs: 180_000,
configOverrides: {
approvals: {
exec: true,
plugin: true,
target: "channel",
},
codexApproval: true,
},
forcedRuntime: "codex",
buildRun: () => ({
approvalKind: "plugin",
appServerMethod: "item/commandExecution/requestApproval",
decision: "allow-once",
kind: "codex-approval",
token: `SLACK_QA_CODEX_EXEC_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`,
}),
},
{
id: "slack-codex-approval-plugin-native",
title: "Slack native Codex file approval prompt resolves",
timeoutMs: 180_000,
configOverrides: {
approvals: {
exec: true,
plugin: true,
target: "channel",
},
codexApproval: true,
},
forcedRuntime: "codex",
buildRun: () => ({
approvalKind: "plugin",
appServerMethod: "item/fileChange/requestApproval",
decision: "allow-once",
kind: "codex-approval",
token: `SLACK_QA_CODEX_FILE_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`,
}),
},
];
export function listSlackQaScenarioCatalog() {
return SLACK_QA_SCENARIOS.map((scenario) => ({ id: scenario.id }));
}
export function getSlackQaScenarioDefinition(id: string) {
const scenario = SLACK_QA_SCENARIOS.find((candidate) => candidate.id === id);
if (!scenario) {
throw new Error(`unknown Slack QA scenario id: ${id}`);
}
return scenario;
}
extensions/qa-lab/src/live-transports/whatsapp/scenario-environment.ts
+65 -57
View File
@@ -11,8 +11,9 @@ import {
resolveWhatsAppQaScenarioTarget,
type WhatsAppObservedMessage,
type WhatsAppQaRuntimeEnv,
type WhatsAppQaScenarioImplementation,
type WhatsAppQaScenarioRun,
} from "./whatsapp-live.contracts.js";
import { getWhatsAppQaScenarioDefinition } from "./whatsapp-live.scenarios.js";
import { waitForWhatsAppChannelStable } from "./whatsapp-live.setup.js";
type AdapterFactory = NonNullable<QaRunnerCliRegistration["adapterFactory"]>;
@@ -20,12 +21,16 @@ type AdapterDefinition = Awaited<ReturnType<AdapterFactory["create"]>>;
type FlowPreparationInput = Parameters<NonNullable<AdapterDefinition["prepareFlow"]>>[0];
export type WhatsAppQaScenarioEnvironment = {
configureScenario: (implementation: WhatsAppQaScenarioImplementation) => Promise<{
run: WhatsAppQaScenarioRun;
}>;
driverAuthDir: string;
gateway: FlowPreparationInput["gateway"];
getDriver: () => WhatsAppQaDriverSession;
observedMessages: WhatsAppObservedMessage[];
replaceDriver: (driver: WhatsAppQaDriverSession) => Promise<void>;
runtimeEnv: WhatsAppQaRuntimeEnv;
scenario: { id: string; timeoutMs: number; title: string };
sutAccountId: string;
sutAuthDir: string;
};
@@ -55,70 +60,73 @@ export function createWhatsAppQaScenarioEnvironment(params: {
const observedMessages: WhatsAppObservedMessage[] = [];
const prepareFlow = async (input: FlowPreparationInput) => {
const scenarioId = input.config.whatsappScenarioId;
if (typeof scenarioId !== "string") {
return undefined;
}
const scenario = getWhatsAppQaScenarioDefinition(scenarioId);
if (scenario.requiresGroupJid && !params.runtimeEnv.groupJid) {
if (params.explicitScenarioSelection) {
throw new Error(
`Requested WhatsApp scenario ${scenario.id} requires groupJid in the credential payload`,
);
}
throw new QaSuiteScenarioSkipError(
`WhatsApp scenario ${scenario.id} requires groupJid in the credential payload`,
);
}
const scenarioRun = scenario.buildRun();
const resolvedTarget = resolveWhatsAppQaScenarioTarget({
groupJid: params.runtimeEnv.groupJid,
scenarioId: scenario.id,
target: scenarioRun.kind === "approval" ? (scenarioRun.target ?? "dm") : scenarioRun.target,
});
const groupJid = resolvedTarget.target === "group" ? resolvedTarget.groupJid : undefined;
const allowFrom =
scenarioRun.kind === "approval"
? [params.runtimeEnv.driverPhoneE164]
: scenarioRun.configMode === "open"
? ["*"]
: scenarioRun.configMode === "pairing"
? ["+15550000000"]
: [params.runtimeEnv.driverPhoneE164];
const dmPolicy =
scenarioRun.kind === "approval"
? "allowlist"
: scenarioRun.configMode === "open" || scenarioRun.configMode === "disabled"
? scenarioRun.configMode
: scenarioRun.configMode === "allowlist"
? "allowlist"
: "pairing";
const snapshot = await readLiveQaGatewayConfig(input.gateway);
const cfg = buildWhatsAppQaConfig(snapshot.config as OpenClawConfig, {
allowFrom,
authDir: params.sutAuthDir,
dmPolicy,
groupJid,
ownerAllowFrom: [params.runtimeEnv.driverPhoneE164],
overrides: scenario.configOverrides,
sutAccountId: params.accountId,
});
await patchLiveQaGatewayConfig({
gateway: input.gateway,
patch: cfg as Record<string, unknown>,
replacePaths: resolveWhatsAppQaReplacePaths(params.accountId),
timeoutMs: input.timeoutMs,
waitForConfigRestartSettle: input.waitForConfigRestartSettle,
});
await waitForWhatsAppChannelStable(input.gateway as never, params.accountId);
return {
whatsappScenarioContext: {
configureScenario: async (implementation: WhatsAppQaScenarioImplementation) => {
if (implementation.requiresGroupJid && !params.runtimeEnv.groupJid) {
if (params.explicitScenarioSelection) {
throw new Error(
`Requested WhatsApp scenario ${input.scenarioId} requires groupJid in the credential payload`,
);
}
throw new QaSuiteScenarioSkipError(
`WhatsApp scenario ${input.scenarioId} requires groupJid in the credential payload`,
);
}
const run = implementation.buildRun();
const resolvedTarget = resolveWhatsAppQaScenarioTarget({
groupJid: params.runtimeEnv.groupJid,
scenarioId: input.scenarioId,
target: run.kind === "approval" ? (run.target ?? "dm") : run.target,
});
const groupJid = resolvedTarget.target === "group" ? resolvedTarget.groupJid : undefined;
const allowFrom =
run.kind === "approval"
? [params.runtimeEnv.driverPhoneE164]
: run.configMode === "open"
? ["*"]
: run.configMode === "pairing"
? ["+15550000000"]
: [params.runtimeEnv.driverPhoneE164];
const dmPolicy =
run.kind === "approval"
? "allowlist"
: run.configMode === "open" || run.configMode === "disabled"
? run.configMode
: run.configMode === "allowlist"
? "allowlist"
: "pairing";
const snapshot = await readLiveQaGatewayConfig(input.gateway);
const cfg = buildWhatsAppQaConfig(snapshot.config as OpenClawConfig, {
allowFrom,
authDir: params.sutAuthDir,
dmPolicy,
groupJid,
ownerAllowFrom: [params.runtimeEnv.driverPhoneE164],
overrides: implementation.configOverrides,
sutAccountId: params.accountId,
});
await patchLiveQaGatewayConfig({
gateway: input.gateway,
patch: cfg as Record<string, unknown>,
replacePaths: resolveWhatsAppQaReplacePaths(params.accountId),
timeoutMs: input.timeoutMs,
waitForConfigRestartSettle: input.waitForConfigRestartSettle,
});
await waitForWhatsAppChannelStable(input.gateway as never, params.accountId);
return { run };
},
driverAuthDir: params.driverAuthDir,
gateway: input.gateway,
getDriver: params.getDriver,
observedMessages,
replaceDriver: params.replaceDriver,
runtimeEnv: params.runtimeEnv,
scenario: {
id: input.scenarioId,
timeoutMs: input.timeoutMs,
title: input.scenarioTitle,
},
sutAccountId: params.accountId,
sutAuthDir: params.sutAuthDir,
} satisfies WhatsAppQaScenarioEnvironment,
extensions/qa-lab/src/live-transports/whatsapp/scenario-runtime.ts
+73 -102
View File
@@ -7,8 +7,10 @@ import {
resolveWhatsAppQaScenarioTarget,
type WhatsAppObservedMessage,
type WhatsAppQaMessageScenarioContext,
type WhatsAppQaScenarioDefinition,
type WhatsAppQaScenarioImplementation,
type WhatsAppQaScenarioMetadata,
type WhatsAppQaScenarioResult,
type WhatsAppQaScenarioRun,
} from "./whatsapp-live.contracts.js";
import {
WHATSAPP_QA_TRANSIENT_DRIVER_ATTEMPTS,
@@ -20,16 +22,64 @@ import {
waitForNoWhatsAppReply,
waitForWhatsAppScenarioSutMessage,
} from "./whatsapp-live.operations.js";
import { getWhatsAppQaScenarioDefinition } from "./whatsapp-live.scenarios.js";
import { waitForWhatsAppChannelStable } from "./whatsapp-live.setup.js";
export {
whatsappQaGroupAudioGatingScenario,
whatsappQaGroupOutboundAudioScenario,
whatsappQaGroupOutboundMediaScenario,
whatsappQaGroupOutboundPollScenario,
whatsappQaInboundStructuredMessagesScenario,
whatsappQaMessageActionsScenario,
whatsappQaOutboundDocumentPreservesFilenameScenario,
whatsappQaOutboundPollScenario,
whatsappQaOutboundSendSerializationScenario,
} from "./whatsapp-live.scenario-implementations.capabilities.js";
export {
whatsappQaBroadcastGroupFanoutScenario,
whatsappQaCanaryScenario,
whatsappQaGroupActivationAlwaysScenario,
whatsappQaGroupPendingHistoryContextScenario,
whatsappQaGroupReplyToBotTriggersScenario,
whatsappQaGroupReplyToMessageScenario,
whatsappQaMentionGatingScenario,
whatsappQaReplyToMessageScenario,
whatsappQaReplyToModeBatchedScenario,
whatsappQaTopLevelReplyShapeScenario,
} from "./whatsapp-live.scenario-implementations.conversation.js";
export {
whatsappQaApprovalExecDenyNativeScenario,
whatsappQaApprovalExecGroupReactionNativeScenario,
whatsappQaApprovalExecNativeScenario,
whatsappQaApprovalExecReactionNativeScenario,
whatsappQaApprovalPluginNativeScenario,
whatsappQaGroupAllowlistBlockScenario,
whatsappQaReplyDeliveryShapeScenario,
whatsappQaStatusReactionLifecycleScenario,
whatsappQaStatusReactionsScenario,
whatsappQaStreamFinalMessageAccountingScenario,
} from "./whatsapp-live.scenario-implementations.delivery.js";
export {
whatsappQaAgentMessageActionReactScenario,
whatsappQaAgentMessageActionUploadFileScenario,
whatsappQaAudioPreflightScenario,
whatsappQaGroupAgentMessageActionReactScenario,
whatsappQaGroupAgentMessageActionUploadFileScenario,
whatsappQaInboundImageCaptionScenario,
whatsappQaInboundReactionNoTriggerScenario,
whatsappQaOutboundMediaMatrixScenario,
whatsappQaReplyContextIsolationScenario,
} from "./whatsapp-live.scenario-implementations.user-path.js";
async function runWhatsAppScenarioAttempt(params: {
environment: WhatsAppQaScenarioEnvironment;
scenario: WhatsAppQaScenarioDefinition;
implementation: WhatsAppQaScenarioImplementation;
run: WhatsAppQaScenarioRun;
scenario: WhatsAppQaScenarioMetadata;
}): Promise<WhatsAppQaScenarioResult> {
const driver = params.environment.getDriver();
const runtimeEnv = params.environment.runtimeEnv;
const scenarioRun = params.scenario.buildRun();
const scenarioRun = params.run;
const resolvedTarget = resolveWhatsAppQaScenarioTarget({
groupJid: runtimeEnv.groupJid,
scenarioId: params.scenario.id,
@@ -61,7 +111,7 @@ async function runWhatsAppScenarioAttempt(params: {
turnSourceTo: approvalTurnSourceTo,
});
return {
...buildWhatsAppQaScenarioResultBase(params.scenario),
...buildWhatsAppQaScenarioResultBase(params.scenario, params.implementation),
status: "pass",
details: `${scenarioRun.approvalKind} approval ${approval.approvalId} resolved ${scenarioRun.decision} in ${approval.rttMs}ms`,
rttMs: approval.rttMs,
@@ -164,7 +214,7 @@ async function runWhatsAppScenarioAttempt(params: {
}),
});
return {
...buildWhatsAppQaScenarioResultBase(params.scenario),
...buildWhatsAppQaScenarioResultBase(params.scenario, params.implementation),
status: "pass",
details: ["no reply", afterSendDetails].filter(Boolean).join("; "),
};
@@ -186,7 +236,7 @@ async function runWhatsAppScenarioAttempt(params: {
const responseObservedAt = new Date(reply.observedAt);
const rttMs = responseObservedAt.getTime() - requestStartedAt.getTime();
return {
...buildWhatsAppQaScenarioResultBase(params.scenario),
...buildWhatsAppQaScenarioResultBase(params.scenario, params.implementation),
status: "pass",
details: [`reply matched in ${rttMs}ms`, afterSendDetails, afterReplyDetails, batchDetails]
.filter(Boolean)
@@ -203,11 +253,23 @@ async function runWhatsAppScenarioAttempt(params: {
};
}
async function runWhatsAppScenario(environment: WhatsAppQaScenarioEnvironment, scenarioId: string) {
const scenario = getWhatsAppQaScenarioDefinition(scenarioId);
export async function runWhatsAppScenario(
environment: WhatsAppQaScenarioEnvironment,
implementation: WhatsAppQaScenarioImplementation,
) {
const scenario = environment.scenario;
const { run: configuredRun } = await environment.configureScenario(implementation);
for (let attempt = 1; attempt <= WHATSAPP_QA_TRANSIENT_DRIVER_ATTEMPTS; attempt += 1) {
try {
const result = await runWhatsAppScenarioAttempt({ environment, scenario });
// Retry with fresh markers and callback state while retaining the gateway config
// prepared from the equivalent first run.
const run = attempt === 1 ? configuredRun : implementation.buildRun();
const result = await runWhatsAppScenarioAttempt({
environment,
implementation,
run,
scenario,
});
return attempt === 1
? result
: { ...result, details: `${result.details}; driver reconnected ${attempt - 1}x` };
@@ -225,96 +287,5 @@ async function runWhatsAppScenario(environment: WhatsAppQaScenarioEnvironment, s
await environment.replaceDriver(nextDriver);
}
}
throw new Error(`WhatsApp scenario ${scenarioId} exhausted driver retries`);
throw new Error(`WhatsApp scenario ${scenario.id} exhausted driver retries`);
}
export const runWhatsAppCanaryScenario = (context: WhatsAppQaScenarioEnvironment) =>
runWhatsAppScenario(context, "whatsapp-canary");
export const runWhatsAppMentionGatingScenario = (context: WhatsAppQaScenarioEnvironment) =>
runWhatsAppScenario(context, "whatsapp-mention-gating");
export const runWhatsAppGroupPendingHistoryContextScenario = (
context: WhatsAppQaScenarioEnvironment,
) => runWhatsAppScenario(context, "whatsapp-group-pending-history-context");
export const runWhatsAppBroadcastGroupFanoutScenario = (context: WhatsAppQaScenarioEnvironment) =>
runWhatsAppScenario(context, "whatsapp-broadcast-group-fanout");
export const runWhatsAppGroupActivationAlwaysScenario = (context: WhatsAppQaScenarioEnvironment) =>
runWhatsAppScenario(context, "whatsapp-group-activation-always");
export const runWhatsAppGroupReplyToBotTriggersScenario = (
context: WhatsAppQaScenarioEnvironment,
) => runWhatsAppScenario(context, "whatsapp-group-reply-to-bot-triggers");
export const runWhatsAppTopLevelReplyShapeScenario = (context: WhatsAppQaScenarioEnvironment) =>
runWhatsAppScenario(context, "whatsapp-top-level-reply-shape");
export const runWhatsAppReplyToMessageScenario = (context: WhatsAppQaScenarioEnvironment) =>
runWhatsAppScenario(context, "whatsapp-reply-to-message");
export const runWhatsAppGroupReplyToMessageScenario = (context: WhatsAppQaScenarioEnvironment) =>
runWhatsAppScenario(context, "whatsapp-group-reply-to-message");
export const runWhatsAppReplyToModeBatchedScenario = (context: WhatsAppQaScenarioEnvironment) =>
runWhatsAppScenario(context, "whatsapp-reply-to-mode-batched");
export const runWhatsAppAgentMessageActionReactScenario = (
context: WhatsAppQaScenarioEnvironment,
) => runWhatsAppScenario(context, "whatsapp-agent-message-action-react");
export const runWhatsAppAgentMessageActionUploadFileScenario = (
context: WhatsAppQaScenarioEnvironment,
) => runWhatsAppScenario(context, "whatsapp-agent-message-action-upload-file");
export const runWhatsAppGroupAgentMessageActionReactScenario = (
context: WhatsAppQaScenarioEnvironment,
) => runWhatsAppScenario(context, "whatsapp-group-agent-message-action-react");
export const runWhatsAppGroupAgentMessageActionUploadFileScenario = (
context: WhatsAppQaScenarioEnvironment,
) => runWhatsAppScenario(context, "whatsapp-group-agent-message-action-upload-file");
export const runWhatsAppInboundReactionNoTriggerScenario = (
context: WhatsAppQaScenarioEnvironment,
) => runWhatsAppScenario(context, "whatsapp-inbound-reaction-no-trigger");
export const runWhatsAppReplyContextIsolationScenario = (context: WhatsAppQaScenarioEnvironment) =>
runWhatsAppScenario(context, "whatsapp-reply-context-isolation");
export const runWhatsAppInboundImageCaptionScenario = (context: WhatsAppQaScenarioEnvironment) =>
runWhatsAppScenario(context, "whatsapp-inbound-image-caption");
export const runWhatsAppAudioPreflightScenario = (context: WhatsAppQaScenarioEnvironment) =>
runWhatsAppScenario(context, "whatsapp-audio-preflight");
export const runWhatsAppOutboundMediaMatrixScenario = (context: WhatsAppQaScenarioEnvironment) =>
runWhatsAppScenario(context, "whatsapp-outbound-media-matrix");
export const runWhatsAppOutboundDocumentPreservesFilenameScenario = (
context: WhatsAppQaScenarioEnvironment,
) => runWhatsAppScenario(context, "whatsapp-outbound-document-preserves-filename");
export const runWhatsAppOutboundSendSerializationScenario = (
context: WhatsAppQaScenarioEnvironment,
) => runWhatsAppScenario(context, "whatsapp-outbound-send-serialization");
export const runWhatsAppOutboundPollScenario = (context: WhatsAppQaScenarioEnvironment) =>
runWhatsAppScenario(context, "whatsapp-outbound-poll");
export const runWhatsAppGroupOutboundMediaScenario = (context: WhatsAppQaScenarioEnvironment) =>
runWhatsAppScenario(context, "whatsapp-group-outbound-media");
export const runWhatsAppGroupOutboundAudioScenario = (context: WhatsAppQaScenarioEnvironment) =>
runWhatsAppScenario(context, "whatsapp-group-outbound-audio");
export const runWhatsAppGroupOutboundPollScenario = (context: WhatsAppQaScenarioEnvironment) =>
runWhatsAppScenario(context, "whatsapp-group-outbound-poll");
export const runWhatsAppMessageActionsScenario = (context: WhatsAppQaScenarioEnvironment) =>
runWhatsAppScenario(context, "whatsapp-message-actions");
export const runWhatsAppInboundStructuredMessagesScenario = (
context: WhatsAppQaScenarioEnvironment,
) => runWhatsAppScenario(context, "whatsapp-inbound-structured-messages");
export const runWhatsAppGroupAudioGatingScenario = (context: WhatsAppQaScenarioEnvironment) =>
runWhatsAppScenario(context, "whatsapp-group-audio-gating");
export const runWhatsAppReplyDeliveryShapeScenario = (context: WhatsAppQaScenarioEnvironment) =>
runWhatsAppScenario(context, "whatsapp-reply-delivery-shape");
export const runWhatsAppStreamFinalMessageAccountingScenario = (
context: WhatsAppQaScenarioEnvironment,
) => runWhatsAppScenario(context, "whatsapp-stream-final-message-accounting");
export const runWhatsAppApprovalExecDenyNativeScenario = (context: WhatsAppQaScenarioEnvironment) =>
runWhatsAppScenario(context, "whatsapp-approval-exec-deny-native");
export const runWhatsAppStatusReactionsScenario = (context: WhatsAppQaScenarioEnvironment) =>
runWhatsAppScenario(context, "whatsapp-status-reactions");
export const runWhatsAppStatusReactionLifecycleScenario = (
context: WhatsAppQaScenarioEnvironment,
) => runWhatsAppScenario(context, "whatsapp-status-reaction-lifecycle");
export const runWhatsAppGroupAllowlistBlockScenario = (context: WhatsAppQaScenarioEnvironment) =>
runWhatsAppScenario(context, "whatsapp-group-allowlist-block");
export const runWhatsAppApprovalExecNativeScenario = (context: WhatsAppQaScenarioEnvironment) =>
runWhatsAppScenario(context, "whatsapp-approval-exec-native");
export const runWhatsAppApprovalExecReactionNativeScenario = (
context: WhatsAppQaScenarioEnvironment,
) => runWhatsAppScenario(context, "whatsapp-approval-exec-reaction-native");
export const runWhatsAppApprovalExecGroupReactionNativeScenario = (
context: WhatsAppQaScenarioEnvironment,
) => runWhatsAppScenario(context, "whatsapp-approval-exec-group-reaction-native");
export const runWhatsAppApprovalPluginNativeScenario = (context: WhatsAppQaScenarioEnvironment) =>
runWhatsAppScenario(context, "whatsapp-approval-plugin-native");
extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.approvals.ts
+3 -3
View File
@@ -16,7 +16,7 @@ import type {
WhatsAppQaApprovalKind,
WhatsAppQaApprovalScenarioRun,
WhatsAppQaGateway,
WhatsAppQaScenarioDefinition,
WhatsAppQaScenarioMetadata,
} from "./whatsapp-live.contracts.js";
import { formatDiagnosticId } from "./whatsapp-live.operations.js";
@@ -210,7 +210,7 @@ async function waitForWhatsAppApprovalMessage(params: {
driver: WhatsAppQaDriverSession;
observedAfter?: Date;
observedMessages: WhatsAppObservedMessage[];
scenario: WhatsAppQaScenarioDefinition;
scenario: WhatsAppQaScenarioMetadata;
state: "pending" | "resolved";
sutPhoneE164: string;
timeoutMs: number;
@@ -267,7 +267,7 @@ export async function runWhatsAppApprovalScenario(params: {
gateway: WhatsAppQaGateway;
observedMessages: WhatsAppObservedMessage[];
run: WhatsAppQaApprovalScenarioRun;
scenario: WhatsAppQaScenarioDefinition;
scenario: WhatsAppQaScenarioMetadata;
sutAccountId: string;
sutPhoneE164: string;
turnSourceTo: string;
extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.contracts.ts
+17 -92
View File
@@ -14,46 +14,6 @@ export type WhatsAppQaRuntimeEnv = {
groupJid?: string;
};
export type WhatsAppQaScenarioId =
| "whatsapp-approval-exec-deny-native"
| "whatsapp-approval-exec-group-reaction-native"
| "whatsapp-approval-exec-reaction-native"
| "whatsapp-agent-message-action-react"
| "whatsapp-agent-message-action-upload-file"
| "whatsapp-audio-preflight"
| "whatsapp-broadcast-group-fanout"
| "whatsapp-canary"
| "whatsapp-group-allowlist-block"
| "whatsapp-group-activation-always"
| "whatsapp-group-agent-message-action-react"
| "whatsapp-group-agent-message-action-upload-file"
| "whatsapp-group-audio-gating"
| "whatsapp-group-outbound-audio"
| "whatsapp-group-outbound-media"
| "whatsapp-group-outbound-poll"
| "whatsapp-group-pending-history-context"
| "whatsapp-group-reply-to-bot-triggers"
| "whatsapp-group-reply-to-message"
| "whatsapp-inbound-reaction-no-trigger"
| "whatsapp-inbound-image-caption"
| "whatsapp-inbound-structured-messages"
| "whatsapp-message-actions"
| "whatsapp-outbound-document-preserves-filename"
| "whatsapp-outbound-media-matrix"
| "whatsapp-outbound-poll"
| "whatsapp-outbound-send-serialization"
| "whatsapp-mention-gating"
| "whatsapp-reply-delivery-shape"
| "whatsapp-reply-context-isolation"
| "whatsapp-reply-to-message"
| "whatsapp-reply-to-mode-batched"
| "whatsapp-stream-final-message-accounting"
| "whatsapp-status-reaction-lifecycle"
| "whatsapp-status-reactions"
| "whatsapp-top-level-reply-shape"
| "whatsapp-approval-exec-native"
| "whatsapp-approval-plugin-native";
export type WhatsAppQaApprovalKind = "exec" | "plugin";
export type WhatsAppQaApprovalDecision = "allow-once" | "deny";
type WhatsAppQaApprovalDecisionMode = "reaction" | "rpc";
@@ -63,47 +23,6 @@ export function toWhatsAppQaError(error: unknown): Error {
return error instanceof Error ? error : new Error(formatErrorMessage(error));
}
const WHATSAPP_QA_SCENARIO_POSTURES = {
"whatsapp-agent-message-action-react": "user-path",
"whatsapp-agent-message-action-upload-file": "user-path",
"whatsapp-approval-exec-deny-native": "native-approval",
"whatsapp-approval-exec-group-reaction-native": "native-approval",
"whatsapp-approval-exec-native": "native-approval",
"whatsapp-approval-exec-reaction-native": "native-approval",
"whatsapp-approval-plugin-native": "native-approval",
"whatsapp-audio-preflight": "user-path",
"whatsapp-broadcast-group-fanout": "user-path",
"whatsapp-canary": "user-path",
"whatsapp-group-activation-always": "user-path",
"whatsapp-group-allowlist-block": "user-path",
"whatsapp-group-agent-message-action-react": "user-path",
"whatsapp-group-agent-message-action-upload-file": "user-path",
"whatsapp-group-audio-gating": "user-path",
"whatsapp-group-outbound-audio": "direct-gateway",
"whatsapp-group-outbound-media": "direct-gateway",
"whatsapp-group-outbound-poll": "direct-gateway",
"whatsapp-group-pending-history-context": "user-path",
"whatsapp-group-reply-to-bot-triggers": "user-path",
"whatsapp-group-reply-to-message": "user-path",
"whatsapp-inbound-image-caption": "user-path",
"whatsapp-inbound-reaction-no-trigger": "user-path",
"whatsapp-inbound-structured-messages": "user-path",
"whatsapp-mention-gating": "user-path",
"whatsapp-message-actions": "direct-gateway",
"whatsapp-outbound-document-preserves-filename": "direct-gateway",
"whatsapp-outbound-media-matrix": "direct-gateway",
"whatsapp-outbound-poll": "direct-gateway",
"whatsapp-outbound-send-serialization": "direct-gateway",
"whatsapp-reply-context-isolation": "direct-gateway",
"whatsapp-reply-delivery-shape": "direct-gateway",
"whatsapp-reply-to-message": "user-path",
"whatsapp-reply-to-mode-batched": "user-path",
"whatsapp-status-reaction-lifecycle": "user-path",
"whatsapp-status-reactions": "user-path",
"whatsapp-stream-final-message-accounting": "user-path",
"whatsapp-top-level-reply-shape": "user-path",
} satisfies Record<WhatsAppQaScenarioId, WhatsAppQaScenarioPosture>;
type WhatsAppQaMessageSendMode =
| {
kind?: "text";
@@ -124,7 +43,7 @@ export type WhatsAppQaGatewayRuntime = Pick<
export type WhatsAppQaGatewayCallContext = {
gateway: Pick<WhatsAppQaGatewayRuntime, "call">;
gatewayTarget: string;
scenarioId: WhatsAppQaScenarioId;
scenarioId: string;
sutAccountId: string;
};
export type WhatsAppQaObservedMessagesContext = {
@@ -145,7 +64,7 @@ export type WhatsAppQaMessageScenarioContext = {
gatewayWorkspaceDir: string;
recordObservedMessage: (message: WhatsAppQaDriverObservedMessage) => void;
requestStartedAt: Date;
scenarioId: WhatsAppQaScenarioId;
scenarioId: string;
scenarioTitle: string;
sent: { messageId?: string };
sutAccountId: string;
@@ -166,7 +85,7 @@ type WhatsAppQaResolvedScenarioTarget =
export function resolveWhatsAppQaScenarioTarget(params: {
groupJid?: string;
scenarioId: WhatsAppQaScenarioId;
scenarioId: string;
target: "dm" | "group";
}): WhatsAppQaResolvedScenarioTarget {
if (params.target === "dm") {
@@ -242,7 +161,7 @@ export type WhatsAppQaApprovalScenarioRun = {
token: string;
};
type WhatsAppQaScenarioRun = WhatsAppQaApprovalScenarioRun | WhatsAppQaMessageScenarioRun;
export type WhatsAppQaScenarioRun = WhatsAppQaApprovalScenarioRun | WhatsAppQaMessageScenarioRun;
export type WhatsAppQaConfigOverrides = {
actions?: boolean;
@@ -263,14 +182,17 @@ export type WhatsAppQaConfigOverrides = {
statusReactions?: boolean;
};
export type WhatsAppQaScenarioDefinition = {
id: WhatsAppQaScenarioId;
title: string;
timeoutMs: number;
export type WhatsAppQaScenarioImplementation = {
buildRun: () => WhatsAppQaScenarioRun;
configOverrides?: WhatsAppQaConfigOverrides;
posture: WhatsAppQaScenarioPosture;
requiresGroupJid?: boolean;
requiredPluginIds?: readonly string[];
};
export type WhatsAppQaScenarioMetadata = {
id: string;
timeoutMs: number;
title: string;
};
export interface WhatsAppObservedMessage extends WhatsAppQaDriverObservedMessage {
@@ -297,10 +219,13 @@ export type WhatsAppQaScenarioResult = {
title: string;
};
export function buildWhatsAppQaScenarioResultBase(scenario: WhatsAppQaScenarioDefinition) {
export function buildWhatsAppQaScenarioResultBase(
scenario: WhatsAppQaScenarioMetadata,
implementation: WhatsAppQaScenarioImplementation,
) {
return {
id: scenario.id,
title: scenario.title,
posture: WHATSAPP_QA_SCENARIO_POSTURES[scenario.id],
posture: implementation.posture,
};
}
extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.gateway.ts
+1 -2
View File
@@ -7,10 +7,9 @@ import type {
WhatsAppQaGatewayCallContext,
WhatsAppQaGatewayRuntime,
WhatsAppQaMessageScenarioContext,
WhatsAppQaScenarioId,
} from "./whatsapp-live.contracts.js";
function buildWhatsAppQaIdempotencyKey(scenarioId: WhatsAppQaScenarioId, label: string) {
function buildWhatsAppQaIdempotencyKey(scenarioId: string, label: string) {
return `${scenarioId}:${label}:${randomUUID()}`;
}
extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.test.ts
+68 -22
View File
@@ -13,12 +13,18 @@ import {
import { describe, expect, it, vi } from "vitest";
import { fingerprintQaCredentialId } from "../../qa-credentials-fingerprint.runtime.js";
import { readQaScenarioById } from "../../scenario-catalog.js";
import { requireFlowScenario } from "../../scenario-catalog.test-utils.js";
import { applyQaMergePatch, collectQaSuiteGatewayConfigPatch } from "../../suite-planning.js";
import { createWhatsAppQaScenarioEnvironment } from "./scenario-environment.js";
import { resolveWhatsAppQaScenarioIds } from "./scenario-selection.js";
import { runWhatsAppApprovalScenario } from "./whatsapp-live.approvals.js";
import { buildWhatsAppQaConfig, parseWhatsAppQaCredentialPayload } from "./whatsapp-live.config.js";
import { resolveWhatsAppQaMessageTargets } from "./whatsapp-live.contracts.js";
import {
resolveWhatsAppQaMessageTargets,
type WhatsAppQaScenarioImplementation,
type WhatsAppQaScenarioMetadata,
type WhatsAppQaScenarioRun,
} from "./whatsapp-live.contracts.js";
import {
callWhatsAppGatewayMessageAction,
callWhatsAppGatewayPoll,
@@ -27,7 +33,10 @@ import {
runWhatsAppStructuredInboundChecks,
waitForScenarioObservedMessage,
} from "./whatsapp-live.operations.js";
import { getWhatsAppQaScenarioDefinition } from "./whatsapp-live.scenarios.js";
import * as whatsappCapabilityScenarios from "./whatsapp-live.scenario-implementations.capabilities.js";
import * as whatsappConversationScenarios from "./whatsapp-live.scenario-implementations.conversation.js";
import * as whatsappDeliveryScenarios from "./whatsapp-live.scenario-implementations.delivery.js";
import * as whatsappUserPathScenarios from "./whatsapp-live.scenario-implementations.user-path.js";
import { unpackWhatsAppAuthArchive } from "./whatsapp-live.setup.js";
const runExecSpy = vi.hoisted(() =>
@@ -108,9 +117,8 @@ function createWhatsAppQaDriverMock(
};
}
type WhatsAppScenarioDefinition = ReturnType<typeof getWhatsAppQaScenarioDefinition>;
type WhatsAppScenarioRun = ReturnType<WhatsAppScenarioDefinition["buildRun"]>;
type WhatsAppMessageScenarioRun = Exclude<WhatsAppScenarioRun, { kind: "approval" }>;
type WhatsAppScenarioDefinition = WhatsAppQaScenarioMetadata & WhatsAppQaScenarioImplementation;
type WhatsAppMessageScenarioRun = Exclude<WhatsAppQaScenarioRun, { kind: "approval" }>;
type WhatsAppScenarioContext = Parameters<NonNullable<WhatsAppMessageScenarioRun["afterSend"]>>[0];
type WhatsAppQaConfigBase = Parameters<typeof testing.buildWhatsAppQaConfig>[0];
type WhatsAppQaConfigParams = Parameters<typeof testing.buildWhatsAppQaConfig>[1];
@@ -159,14 +167,44 @@ function buildWhatsAppQaConfigFixture(
});
}
type WhatsAppScenarioIdFilter = Parameters<typeof getWhatsAppQaScenarioDefinition>[0];
type WhatsAppScenarioIdFilter = string;
const whatsappScenarioImplementations = {
...whatsappCapabilityScenarios,
...whatsappConversationScenarios,
...whatsappDeliveryScenarios,
...whatsappUserPathScenarios,
} as Record<string, WhatsAppQaScenarioImplementation>;
function toWhatsAppScenarioExportName(id: string) {
const suffix = id
.slice("whatsapp-".length)
.split("-")
.map((part) => `${part.slice(0, 1).toUpperCase()}${part.slice(1)}`)
.join("");
return `whatsappQa${suffix}Scenario`;
}
function getWhatsAppScenario(id: string): WhatsAppScenarioDefinition {
const implementation = whatsappScenarioImplementations[toWhatsAppScenarioExportName(id)];
if (!implementation) {
throw new Error(`missing WhatsApp test implementation for ${id}`);
}
const scenario = requireFlowScenario(readQaScenarioById(id));
return {
...implementation,
id,
timeoutMs: scenario.execution.timeoutMs ?? 60_000,
title: scenario.title,
};
}
function findScenarios(ids: readonly string[]) {
return ids.map((id) => getWhatsAppQaScenarioDefinition(id));
return ids.map(getWhatsAppScenario);
}
function findWhatsAppScenario(id: WhatsAppScenarioIdFilter) {
return getWhatsAppQaScenarioDefinition(id);
return getWhatsAppScenario(id);
}
function updateObservedMessage(
@@ -202,7 +240,7 @@ const WHATSAPP_GROUP_CAPABILITY_SCENARIO_IDS = [
] as const satisfies readonly WhatsAppScenarioIdFilter[];
function findMockWhatsAppScenario(id: WhatsAppScenarioIdFilter) {
const scenario = getWhatsAppQaScenarioDefinition(id);
const scenario = getWhatsAppScenario(id);
const mockScenarioIds = new Set(resolveWhatsAppQaScenarioIds({ providerMode: "mock-openai" }));
if (!mockScenarioIds.has(id)) {
throw new Error(`missing WhatsApp mock-openai scenario ${id}`);
@@ -1114,14 +1152,19 @@ describe("WhatsApp QA live runtime", () => {
sutAuthDir: "/tmp/whatsapp-sut",
});
await prepareFlow({
config: { whatsappScenarioId: "whatsapp-canary" },
const prepared = await prepareFlow({
config: {},
gateway: { call: gatewayCall } as never,
outputDir: "/tmp/whatsapp-output",
primaryModel: "mock-openai/gpt-5.6-luna",
scenarioId: "whatsapp-canary",
scenarioTitle: "WhatsApp DM canary",
timeoutMs: 60_000,
waitForConfigRestartSettle: vi.fn(),
});
await prepared.whatsappScenarioContext.configureScenario(
findWhatsAppScenario("whatsapp-canary"),
);
const patchCall = gatewayCall.mock.calls.find(([method]) => method === "config.patch");
if (!patchCall) {
@@ -1152,16 +1195,19 @@ describe("WhatsApp QA live runtime", () => {
sutAuthDir: "/tmp/whatsapp-sut",
});
await expect(
prepareFlow({
config: { policyKey: "dmPolicy", policyValue: "disabled" },
gateway: { call: gatewayCall } as never,
outputDir: "/tmp/whatsapp-output",
primaryModel: "mock-openai/gpt-5.6-luna",
timeoutMs: 60_000,
waitForConfigRestartSettle: vi.fn(),
}),
).resolves.toBeUndefined();
const prepared = await prepareFlow({
config: { policyKey: "dmPolicy", policyValue: "disabled" },
gateway: { call: gatewayCall } as never,
outputDir: "/tmp/whatsapp-output",
primaryModel: "mock-openai/gpt-5.6-luna",
scenarioId: "whatsapp-access-control-dm-disabled",
scenarioTitle: "WhatsApp dmPolicy disabled stays quiet",
timeoutMs: 60_000,
waitForConfigRestartSettle: vi.fn(),
});
expect(prepared.whatsappScenarioContext.scenario.id).toBe(
"whatsapp-access-control-dm-disabled",
);
expect(gatewayCall).not.toHaveBeenCalled();
});
@@ -2084,7 +2130,7 @@ describe("WhatsApp QA live runtime", () => {
throw new Error("whatsapp-audio-preflight unexpectedly built an approval scenario run");
}
expect(scenario.requiredPluginIds).toEqual(["openai"]);
expect(readQaScenarioById(scenario.id).plugins).toEqual(["openai"]);
expect(scenarioRun.expectReply).toBe(true);
expect(scenarioRun.matchText).toBe("WHATSAPP_QA_AUDIO_TRANSCRIPT_OK");
expect(scenarioRun.sendMode).toMatchObject({
extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenario-implementations.capabilities.ts
+408
View File
@@ -0,0 +1,408 @@
// QA Lab WhatsApp Gateway capability and structured-message scenarios.
import { randomUUID } from "node:crypto";
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
import type { WhatsAppQaScenarioImplementation } from "./whatsapp-live.contracts.js";
import {
WHATSAPP_QA_AUDIO_OGG_OPUS_MIME,
WHATSAPP_QA_GROUP_AUDIO_TRANSCRIPT_MARKER,
WHATSAPP_QA_ONE_PIXEL_PNG,
callWhatsAppGatewayMessageAction,
callWhatsAppGatewayPoll,
callWhatsAppGatewaySend,
callWhatsAppGatewaySendConcurrently,
createWhatsAppQaAudioOggOpusBuffer,
createWhatsAppQaPdfBuffer,
requireWhatsAppTriggerMessageId,
runWhatsAppStructuredInboundChecks,
waitForScenarioObservedMessage,
waitForWhatsAppScenarioSutMessage,
waitForWhatsAppSutReactionToTrigger,
writeWhatsAppQaWorkspaceFixture,
} from "./whatsapp-live.operations.js";
export const whatsappQaOutboundDocumentPreservesFilenameScenario: WhatsAppQaScenarioImplementation =
{
posture: "direct-gateway",
buildRun: () => {
const token = `WHATSAPP_QA_DOCUMENT_FILE_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
afterReply: async (_reply, context) => {
const documentPath = await writeWhatsAppQaWorkspaceFixture(context, {
buffer: createWhatsAppQaPdfBuffer(),
fileName: `whatsapp-qa-report-${token}.pdf`,
});
const documentStartedAt = new Date();
await callWhatsAppGatewaySend(context, {
forceDocument: true,
label: "document-filename",
mediaUrl: documentPath,
message: `${token}_CAPTION`,
});
const document = await waitForScenarioObservedMessage(context, {
observedAfter: documentStartedAt,
match: (message) =>
message.kind === "media" &&
message.hasMedia === true &&
message.text.includes(`${token}_CAPTION`) &&
message.mediaFileName === `whatsapp-qa-report-${token}.pdf`,
});
return `document ${document.mediaFileName ?? "<missing filename>"} preserved`;
},
configMode: "allowlist",
expectReply: true,
input: `Reply with only this exact marker before document filename check: ${token}`,
matchText: token,
target: "dm",
};
},
};
export const whatsappQaOutboundSendSerializationScenario: WhatsAppQaScenarioImplementation = {
posture: "direct-gateway",
buildRun: () => {
const token = `WHATSAPP_QA_SERIAL_SEND_${randomUUID().slice(0, 8).toUpperCase()}`;
const markers = Array.from({ length: 5 }, (_, index) => `${token}_${index + 1}`);
return {
afterReply: async (_reply, context) => {
const sendsStartedAt = new Date();
await callWhatsAppGatewaySendConcurrently(
context,
markers.map((marker, index) => ({
label: `parallel-${index + 1}`,
message: marker,
})),
);
await Promise.all(
markers.map((marker) =>
waitForScenarioObservedMessage(context, {
observedAfter: sendsStartedAt,
match: (message) => message.kind === "text" && message.text.includes(marker),
}),
),
);
return `gateway parallel send delivered ${markers.length}/${markers.length} messages`;
},
configMode: "allowlist",
expectReply: true,
input: `Reply with only this exact marker before parallel send checks: ${token}`,
matchText: token,
target: "dm",
};
},
};
export const whatsappQaOutboundPollScenario: WhatsAppQaScenarioImplementation = {
posture: "direct-gateway",
buildRun: () => {
const token = `WHATSAPP_QA_OUTBOUND_POLL_${randomUUID().slice(0, 8).toUpperCase()}`;
const question = `${token} choose one`;
return {
afterReply: async (_reply, context) => {
const pollStartedAt = new Date();
await callWhatsAppGatewayPoll(context, {
label: "poll",
options: ["alpha", "beta"],
question,
});
const poll = await waitForScenarioObservedMessage(context, {
observedAfter: pollStartedAt,
match: (message) =>
message.kind === "poll" &&
message.poll?.question === question &&
message.poll.options.includes("alpha") &&
message.poll.options.includes("beta"),
});
return `poll observed with ${poll.poll?.options.length ?? 0} options`;
},
configMode: "allowlist",
expectReply: true,
input: `Reply with only this exact marker before outbound poll check: ${token}`,
matchText: token,
target: "dm",
};
},
};
export const whatsappQaGroupOutboundMediaScenario: WhatsAppQaScenarioImplementation = {
posture: "direct-gateway",
requiresGroupJid: true,
buildRun: () => {
const token = `WHATSAPP_QA_GROUP_OUTBOUND_MEDIA_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
afterReply: async (_reply, context) => {
const mediaRootToken = randomUUID().slice(0, 8);
const imagePath = await writeWhatsAppQaWorkspaceFixture(context, {
buffer: WHATSAPP_QA_ONE_PIXEL_PNG,
fileName: `whatsapp-qa-group-${mediaRootToken}.png`,
});
const documentPath = await writeWhatsAppQaWorkspaceFixture(context, {
buffer: createWhatsAppQaPdfBuffer(),
fileName: `whatsapp-qa-group-${mediaRootToken}.pdf`,
});
const imageStartedAt = new Date();
await callWhatsAppGatewaySend(context, {
label: "group-image",
mediaUrl: imagePath,
message: `${token}_IMAGE`,
});
await waitForWhatsAppScenarioSutMessage(context, {
observedAfter: imageStartedAt,
targetKind: "group",
match: (message) =>
message.kind === "media" &&
message.hasMedia === true &&
message.mediaType?.startsWith("image/") === true &&
message.text.includes(`${token}_IMAGE`),
});
const documentStartedAt = new Date();
await callWhatsAppGatewaySend(context, {
forceDocument: true,
label: "group-document",
mediaUrl: documentPath,
message: `${token}_DOCUMENT`,
});
await waitForWhatsAppScenarioSutMessage(context, {
observedAfter: documentStartedAt,
targetKind: "group",
match: (message) =>
message.kind === "media" &&
message.hasMedia === true &&
(message.mediaType === "application/pdf" ||
message.mediaFileName?.endsWith(".pdf") === true) &&
message.text.includes(`${token}_DOCUMENT`),
});
return "gateway send delivered image and document media to the group";
},
configMode: "allowlist",
expectReply: true,
input: `openclawqa reply with only this exact marker before group outbound media checks: ${token}`,
matchText: token,
target: "group",
};
},
};
export const whatsappQaGroupOutboundAudioScenario: WhatsAppQaScenarioImplementation = {
posture: "direct-gateway",
requiresGroupJid: true,
buildRun: () => {
const token = `WHATSAPP_QA_GROUP_OUTBOUND_AUDIO_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
afterReply: async (_reply, context) => {
const audioPath = await writeWhatsAppQaWorkspaceFixture(context, {
buffer: createWhatsAppQaAudioOggOpusBuffer({ variant: "group-trigger" }),
fileName: `whatsapp-qa-group-audio-${token}.ogg`,
});
const audioStartedAt = new Date();
await callWhatsAppGatewaySend(context, {
asVoice: true,
label: "group-audio",
mediaUrl: audioPath,
message: `${token}_AUDIO`,
});
await waitForWhatsAppScenarioSutMessage(context, {
observedAfter: audioStartedAt,
targetKind: "group",
match: (message) =>
message.kind === "media" &&
message.hasMedia === true &&
message.mediaType?.startsWith("audio/") === true,
});
await waitForWhatsAppScenarioSutMessage(context, {
observedAfter: audioStartedAt,
targetKind: "group",
match: (message) => message.text.includes(`${token}_AUDIO`),
});
return "gateway send delivered audio media to the group";
},
configMode: "allowlist",
expectReply: true,
input: `openclawqa reply with only this exact marker before group outbound audio check: ${token}`,
matchText: token,
target: "group",
};
},
};
export const whatsappQaGroupOutboundPollScenario: WhatsAppQaScenarioImplementation = {
posture: "direct-gateway",
requiresGroupJid: true,
buildRun: () => {
const token = `WHATSAPP_QA_GROUP_OUTBOUND_POLL_${randomUUID().slice(0, 8).toUpperCase()}`;
const question = `${token} choose one`;
return {
afterReply: async (_reply, context) => {
const pollStartedAt = new Date();
await callWhatsAppGatewayPoll(context, {
label: "group-poll",
options: ["alpha", "beta"],
question,
});
const poll = await waitForWhatsAppScenarioSutMessage(context, {
observedAfter: pollStartedAt,
targetKind: "group",
match: (message) =>
message.kind === "poll" &&
message.poll?.question === question &&
message.poll.options.includes("alpha") &&
message.poll.options.includes("beta"),
});
return `group poll observed with ${poll.poll?.options.length ?? 0} options`;
},
configMode: "allowlist",
expectReply: true,
input: `openclawqa reply with only this exact marker before group outbound poll check: ${token}`,
matchText: token,
target: "group",
};
},
};
export const whatsappQaMessageActionsScenario: WhatsAppQaScenarioImplementation = {
posture: "direct-gateway",
configOverrides: {
actions: true,
},
buildRun: () => {
const token = `WHATSAPP_QA_ACTIONS_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
afterReply: async (_reply, context) => {
const triggerMessageId = requireWhatsAppTriggerMessageId(context);
const reactionStartedAt = new Date();
await callWhatsAppGatewayMessageAction(context, {
action: "react",
label: "react",
params: {
emoji: "👍",
messageId: triggerMessageId,
},
});
await waitForWhatsAppSutReactionToTrigger(context, {
expectation: { emoji: "👍" },
observedAfter: reactionStartedAt,
});
const uploadStartedAt = new Date();
await callWhatsAppGatewayMessageAction(context, {
action: "upload-file",
label: "upload-file",
params: {
buffer: WHATSAPP_QA_ONE_PIXEL_PNG.toString("base64"),
caption: `${token}_UPLOAD`,
contentType: "image/png",
filename: "whatsapp-qa-upload.png",
},
});
await waitForScenarioObservedMessage(context, {
observedAfter: uploadStartedAt,
match: (message) =>
message.kind === "media" &&
message.mediaType?.startsWith("image/") === true &&
message.text.includes(`${token}_UPLOAD`),
});
return "message.action react and upload-file observed";
},
configMode: "allowlist",
expectReply: true,
input: `Reply with only this exact marker before action checks: ${token}`,
matchText: token,
target: "dm",
};
},
};
export const whatsappQaInboundStructuredMessagesScenario: WhatsAppQaScenarioImplementation = {
posture: "user-path",
buildRun: () => {
const token = `WHATSAPP_QA_STRUCTURED_${randomUUID().slice(0, 8).toUpperCase()}`;
const locationToken = `${token}_LOCATION`;
const contactToken = `${token}_CONTACT`;
const stickerToken = `${token}_STICKER`;
const locationCoordinateText = "37.774900, -122.419400";
return {
afterReply: async (_reply, context) => {
const waitForStructuredReply = async (
label: string,
observedAfter: Date,
expectedToken: string,
) => {
try {
return await waitForScenarioObservedMessage(context, {
observedAfter,
timeoutMs: 60_000,
match: (message) => message.text.includes(expectedToken),
diagnosticChecks: [
{
label: "containsExpectedToken",
match: (message) => message.text.includes(expectedToken),
},
],
});
} catch (error) {
throw new Error(
`timed out waiting for WhatsApp structured ${label} reply (${expectedToken}): ${formatErrorMessage(error)}`,
{ cause: error },
);
}
};
await runWhatsAppStructuredInboundChecks({
contactToken,
documentToken: `${token}_DOCUMENT`,
driver: context.driver,
driverPhoneE164: context.driverPhoneE164,
locationToken,
stickerToken,
target: context.target,
waitForStructuredReply,
});
return "document, location, contact, and sticker elicited replies";
},
configMode: "allowlist",
expectReply: true,
input:
`When a later WhatsApp location message shows ${locationCoordinateText}, ` +
`reply with only this WhatsApp location marker: ${locationToken}. ` +
`When a later WhatsApp contact message appears, ` +
`reply with only this WhatsApp contact marker: ${contactToken}. ` +
`When a later WhatsApp sticker message appears, ` +
`reply with only this WhatsApp sticker marker: ${stickerToken}. ` +
`Reply with only this exact marker before structured inbound checks: ${token}`,
matchText: token,
target: "dm",
};
},
};
export const whatsappQaGroupAudioGatingScenario: WhatsAppQaScenarioImplementation = {
posture: "user-path",
configOverrides: {
audioPreflight: true,
},
requiresGroupJid: true,
buildRun: () => ({
configMode: "allowlist",
expectReply: true,
input: "",
matchText: WHATSAPP_QA_GROUP_AUDIO_TRANSCRIPT_MARKER,
quietInput: "",
quietSendMode: {
fileName: "whatsapp-qa-group-audio-quiet.ogg",
kind: "media",
mediaBuffer: createWhatsAppQaAudioOggOpusBuffer(),
mediaType: WHATSAPP_QA_AUDIO_OGG_OPUS_MIME,
},
quietWindowMs: 5_000,
sendMode: {
fileName: "whatsapp-qa-group-audio.ogg",
kind: "media",
mediaBuffer: createWhatsAppQaAudioOggOpusBuffer({
variant: "group-trigger",
}),
mediaType: WHATSAPP_QA_AUDIO_OGG_OPUS_MIME,
},
target: "group",
}),
};
extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenario-implementations.conversation.ts
+366
View File
@@ -0,0 +1,366 @@
// QA Lab WhatsApp conversation and reply-context scenarios.
import { randomUUID } from "node:crypto";
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
import {
toWhatsAppQaError,
type WhatsAppQaMessageScenarioRun,
type WhatsAppQaScenarioImplementation,
} from "./whatsapp-live.contracts.js";
import {
assertWhatsAppMessageFromSutPhone,
assertWhatsAppMessagesFromSutPhone,
buildWhatsAppQuotedMessageKeyFromObservedMessage,
resolveWhatsAppQaNoReplyTarget,
waitForDistinctWhatsAppSutMessages,
waitForNoWhatsAppReply,
waitForWhatsAppScenarioSutMessage,
} from "./whatsapp-live.operations.js";
function buildWhatsAppQuoteReplyRun(target: "dm" | "group"): WhatsAppQaMessageScenarioRun {
const token = `WHATSAPP_QA_REPLY_TO_${target.toUpperCase()}_${randomUUID().slice(0, 8).toUpperCase()}`;
const input =
target === "group"
? `openclawqa reply with only this exact marker: ${token}`
: `Reply with only this exact marker: ${token}`;
return {
configMode: "allowlist",
expectReply: true,
input,
matchText: token,
target,
verify: (reply, context) => {
if (!context.sent.messageId) {
throw new Error("WhatsApp driver did not return a triggering message id.");
}
if (reply.quoted?.messageId !== context.sent.messageId) {
throw new Error(
`expected reply quote ${context.sent.messageId}, got ${reply.quoted?.messageId ?? "<missing>"}`,
);
}
},
};
}
export const whatsappQaCanaryScenario: WhatsAppQaScenarioImplementation = {
posture: "user-path",
buildRun: () => {
const token = `WHATSAPP_QA_ECHO_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
configMode: "allowlist",
expectReply: true,
input: `Reply with only this exact marker: ${token}`,
matchText: token,
target: "dm",
};
},
};
export const whatsappQaMentionGatingScenario: WhatsAppQaScenarioImplementation = {
posture: "user-path",
requiresGroupJid: true,
buildRun: () => {
const quietToken = `WHATSAPP_QA_GROUP_QUIET_${randomUUID().slice(0, 8).toUpperCase()}`;
const replyToken = `WHATSAPP_QA_GROUP_MENTION_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
configMode: "allowlist",
expectReply: true,
input: `openclawqa reply with only this exact marker: ${replyToken}`,
matchText: replyToken,
quietInput: `This group message is intentionally unmentioned. If you respond, include ${quietToken}.`,
quietMatchText: quietToken,
quietWindowMs: 5_000,
target: "group",
};
},
};
export const whatsappQaGroupPendingHistoryContextScenario: WhatsAppQaScenarioImplementation = {
posture: "user-path",
configOverrides: {
groupHistoryLimit: 50,
groupPolicy: "open",
inboundDebounceMs: 0,
replyToMode: "all",
},
requiresGroupJid: true,
buildRun: () => {
const suffix = randomUUID().slice(0, 8).toUpperCase();
const quietMarker = `WHATSAPP_QA_PENDING_HISTORY_QUIET_${suffix}`;
const contextSentinel = `WHATSAPP_QA_PENDING_HISTORY_CONTEXT_ONLY_${suffix}`;
const triggerMarker = `WHATSAPP_QA_PENDING_HISTORY_TRIGGER_${suffix}`;
const okMarker = `WHATSAPP_QA_PENDING_HISTORY_OK_${suffix}`;
return {
configMode: "open",
expectReply: true,
expectedSutMessageCount: 1,
input:
`openclawqa pending history context check ${triggerMarker}. ` +
`Reply with only ${okMarker} only if the previous quiet group message is present ` +
`in prior group context with its context-only sentinel. ` +
"Do not use current-message text as proof.",
matchText: okMarker,
quietInput: `quiet context marker ${quietMarker} ${contextSentinel}`,
quietWindowMs: 5_000,
target: "group",
};
},
};
export const whatsappQaBroadcastGroupFanoutScenario: WhatsAppQaScenarioImplementation = {
posture: "user-path",
configOverrides: {
broadcast: {
agents: ["main", "qa-second"],
strategy: "sequential",
},
groupPolicy: "open",
},
requiresGroupJid: true,
buildRun: () => {
const token = `WHATSAPP_QA_BROADCAST_TOKEN_${randomUUID().slice(0, 8).toUpperCase()}`;
const mainMarker = `${token}_MAIN`;
const secondMarker = `${token}_SECOND`;
return {
afterReply: async (reply, context) => {
const replies = await waitForDistinctWhatsAppSutMessages(context, {
initialMessages: [reply],
matchers: [
(message) => message.text.includes(mainMarker),
(message) => message.text.includes(secondMarker),
],
observedAfter: context.requestStartedAt,
timeoutMs: 60_000,
});
assertWhatsAppMessagesFromSutPhone(replies, context);
return "broadcast fanout produced main and qa-second replies";
},
configMode: "open",
expectReply: true,
input: `openclawqa broadcast fanout check ${token}`,
matchText: mainMarker,
target: "group",
};
},
};
export const whatsappQaGroupActivationAlwaysScenario: WhatsAppQaScenarioImplementation = {
posture: "user-path",
configOverrides: {
groupPolicy: "open",
},
requiresGroupJid: true,
buildRun: () => {
const suffix = randomUUID().slice(0, 8).toUpperCase();
const alwaysMarker = `WHATSAPP_QA_ACTIVATION_ALWAYS_${suffix}`;
const quietMarker = `WHATSAPP_QA_ACTIVATION_QUIET_${suffix}`;
return {
afterReply: async (reply, context) => {
assertWhatsAppMessageFromSutPhone(reply, context);
let activationProbeError: unknown;
try {
const alwaysStartedAt = new Date();
await context.driver.sendText(
context.target,
`Group activation visible behavior marker ${alwaysMarker}`,
);
const alwaysReply = await waitForWhatsAppScenarioSutMessage(context, {
match: (message) => message.text.includes(alwaysMarker),
observedAfter: alwaysStartedAt,
targetKind: "group",
timeoutMs: 60_000,
});
assertWhatsAppMessageFromSutPhone(alwaysReply, context);
} catch (error) {
activationProbeError = error;
}
let restoreError: unknown;
const restoreStartedAt = new Date();
try {
await context.driver.sendText(context.target, "/activation mention");
const restoreReply = await waitForWhatsAppScenarioSutMessage(context, {
match: (message) => /\bactivation\b.*\bmention\b/iu.test(message.text),
observedAfter: restoreStartedAt,
targetKind: "group",
timeoutMs: 60_000,
});
assertWhatsAppMessageFromSutPhone(restoreReply, context);
} catch (error) {
restoreError = error;
}
if (activationProbeError && restoreError) {
throw new Error(
`activation always probe failed; additionally failed to restore mention mode: ${formatErrorMessage(restoreError)}`,
{ cause: activationProbeError },
);
}
if (activationProbeError) {
throw toWhatsAppQaError(activationProbeError);
}
if (restoreError) {
throw toWhatsAppQaError(restoreError);
}
const quietStartedAt = new Date();
await context.driver.sendText(
context.target,
`Group activation quiet marker ${quietMarker}`,
);
await waitForNoWhatsAppReply({
driver: context.driver,
observedAfter: quietStartedAt,
sutPhoneE164: context.sutPhoneE164,
windowMs: 5_000,
...resolveWhatsAppQaNoReplyTarget({
groupJid: context.target,
target: "group",
}),
});
return "activation always replied to an unmentioned group message and mention mode was restored";
},
configMode: "allowlist",
expectReply: true,
input: "/activation always",
matchText: /\bactivation\b.*\balways\b/iu,
target: "group",
};
},
};
export const whatsappQaGroupReplyToBotTriggersScenario: WhatsAppQaScenarioImplementation = {
posture: "user-path",
configOverrides: {
groupPolicy: "open",
},
requiresGroupJid: true,
buildRun: () => {
const suffix = randomUUID().slice(0, 8).toUpperCase();
const seedMarker = `WHATSAPP_QA_REPLY_TO_BOT_SEED_${suffix}`;
const triggerMarker = `WHATSAPP_QA_REPLY_TO_BOT_TRIGGER_${suffix}`;
return {
afterReply: async (reply, context) => {
assertWhatsAppMessageFromSutPhone(reply, context);
const quotedStartedAt = new Date();
const quotedTrigger = await context.driver.sendText(
context.target,
`Quoted implicit reply trigger marker ${triggerMarker}`,
{
quotedMessageKey: buildWhatsAppQuotedMessageKeyFromObservedMessage(reply, {
remoteJid: context.target,
}),
},
);
if (!quotedTrigger.messageId) {
throw new Error("WhatsApp driver did not return a quoted trigger message id.");
}
const quotedTriggerMessageId = quotedTrigger.messageId;
const quotedReply = await waitForWhatsAppScenarioSutMessage(context, {
diagnosticChecks: [
{
label: "containsTriggerMarker",
match: (message) => message.text.includes(triggerMarker),
},
{
label: "quotesTrigger",
match: (message) => message.quoted?.messageId === quotedTriggerMessageId,
},
],
match: (message) =>
message.text.includes(triggerMarker) &&
message.quoted?.messageId === quotedTriggerMessageId,
observedAfter: quotedStartedAt,
targetKind: "group",
timeoutMs: 60_000,
});
assertWhatsAppMessageFromSutPhone(quotedReply, context);
return "quoted reply to bot triggered a group response without an explicit mention";
},
configMode: "allowlist",
expectReply: true,
input: `openclawqa Mentioned group seed marker ${seedMarker}`,
matchText: seedMarker,
target: "group",
};
},
};
export const whatsappQaTopLevelReplyShapeScenario: WhatsAppQaScenarioImplementation = {
posture: "user-path",
configOverrides: {
replyToMode: "off",
},
buildRun: () => {
const token = `WHATSAPP_QA_TOP_LEVEL_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
configMode: "allowlist",
expectReply: true,
input: `Reply with only this exact marker: ${token}`,
matchText: token,
target: "dm",
verify: (reply) => {
if (reply.quoted?.messageId) {
throw new Error(
`expected top-level WhatsApp reply without quote metadata, got quoted message ${reply.quoted.messageId}`,
);
}
},
};
},
};
export const whatsappQaReplyToMessageScenario: WhatsAppQaScenarioImplementation = {
posture: "user-path",
configOverrides: {
replyToMode: "all",
},
buildRun: () => buildWhatsAppQuoteReplyRun("dm"),
};
export const whatsappQaGroupReplyToMessageScenario: WhatsAppQaScenarioImplementation = {
posture: "user-path",
configOverrides: {
replyToMode: "all",
},
requiresGroupJid: true,
buildRun: () => buildWhatsAppQuoteReplyRun("group"),
};
export const whatsappQaReplyToModeBatchedScenario: WhatsAppQaScenarioImplementation = {
posture: "user-path",
configOverrides: {
inboundDebounceMs: 250,
replyToMode: "batched",
},
buildRun: () => {
const suffix = randomUUID().slice(0, 8).toUpperCase();
const firstToken = `WHATSAPP_QA_BATCHED_FIRST_${suffix}`;
const finalToken = `WHATSAPP_QA_BATCHED_FINAL_${suffix}`;
let secondMessageId: string | undefined;
return {
afterSend: async (context) => {
const second = await context.driver.sendText(
context.target,
`Second batched WhatsApp QA message. Reply with only this exact marker: ${finalToken} only if the previous queued message is visible in this same run context.`,
);
secondMessageId = second.messageId;
return "second batched message sent before debounce flush";
},
configMode: "allowlist",
expectReply: true,
input: `First batched WhatsApp QA message ${firstToken}. Wait for the next message before replying.`,
matchText: finalToken,
target: "dm",
verify: (reply) => {
if (!secondMessageId) {
throw new Error("WhatsApp driver did not return a second batched message id.");
}
if (reply.quoted?.messageId !== secondMessageId) {
throw new Error(
`expected batched reply quote ${secondMessageId}, got ${reply.quoted?.messageId ?? "<missing>"}`,
);
}
},
};
},
};
extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenario-implementations.delivery.ts
+237
View File
@@ -0,0 +1,237 @@
// QA Lab WhatsApp delivery-shape, status, and approval scenarios.
import { randomUUID } from "node:crypto";
import type { WhatsAppQaScenarioImplementation } from "./whatsapp-live.contracts.js";
import {
callWhatsAppGatewaySend,
waitForScenarioObservedMessage,
waitForWhatsAppSutReactionSequenceToTrigger,
waitForWhatsAppSutReactionToTrigger,
} from "./whatsapp-live.operations.js";
export const whatsappQaReplyDeliveryShapeScenario: WhatsAppQaScenarioImplementation = {
posture: "direct-gateway",
buildRun: () => {
const token = `WHATSAPP_QA_REPLY_SHAPE_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
afterReply: async (_reply, context) => {
if (!context.sent.messageId) {
throw new Error("WhatsApp driver did not return a triggering message id.");
}
const quotedTriggerMessageId = context.sent.messageId;
const chunkStartedAt = new Date();
const longText = `${token}_LONG_BEGIN\n${"A".repeat(4_500)}\n${token}_LONG_END`;
await callWhatsAppGatewaySend(context, {
label: "long-reply",
message: longText,
replyToId: quotedTriggerMessageId,
});
const firstChunk = await waitForScenarioObservedMessage(context, {
observedAfter: chunkStartedAt,
diagnosticChecks: [
{
label: "longBeginMarker",
match: (message) => message.text.includes(`${token}_LONG_BEGIN`),
},
{
label: "quotesTrigger",
match: (message) => message.quoted?.messageId === quotedTriggerMessageId,
},
],
match: (message) =>
message.text.includes(`${token}_LONG_BEGIN`) &&
message.quoted?.messageId === quotedTriggerMessageId,
});
const secondChunk = await waitForScenarioObservedMessage(context, {
observedAfter: chunkStartedAt,
diagnosticChecks: [
{
label: "longEndMarker",
match: (message) => message.text.includes(`${token}_LONG_END`),
},
{
label: "quotesTrigger",
match: (message) => message.quoted?.messageId === quotedTriggerMessageId,
},
],
match: (message) =>
message.messageId !== firstChunk.messageId &&
message.text.includes(`${token}_LONG_END`) &&
message.quoted?.messageId === quotedTriggerMessageId,
});
return `long reply chunked across ${firstChunk.messageId ?? "<first>"} and ${secondChunk.messageId ?? "<second>"}`;
},
configMode: "allowlist",
expectReply: true,
input: `Reply with only this exact marker before reply-shape checks: ${token}`,
matchText: token,
target: "dm",
};
},
};
export const whatsappQaStreamFinalMessageAccountingScenario: WhatsAppQaScenarioImplementation = {
posture: "user-path",
buildRun: () => ({
configMode: "allowlist",
expectReply: true,
expectedJoinedSutTextIncludes: ["WHATSAPP-LONG-FINAL-BEGIN", "WHATSAPP-LONG-FINAL-END"],
expectedSutMessageCount: 2,
input: "WhatsApp long final QA check. Use the scripted long final response.",
matchText: "WHATSAPP-LONG-FINAL-BEGIN",
settleMs: 4_000,
target: "dm",
}),
};
export const whatsappQaApprovalExecDenyNativeScenario: WhatsAppQaScenarioImplementation = {
posture: "native-approval",
configOverrides: {
approvals: {
exec: true,
},
},
buildRun: () => ({
approvalKind: "exec",
decision: "deny",
kind: "approval",
token: `WHATSAPP_QA_EXEC_DENY_${randomUUID().slice(0, 8).toUpperCase()}`,
}),
};
export const whatsappQaStatusReactionsScenario: WhatsAppQaScenarioImplementation = {
posture: "user-path",
configOverrides: {
statusReactions: true,
},
buildRun: () => {
const token = `WHATSAPP_QA_STATUS_REACTION_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
afterSend: async (context) => {
const reaction = await waitForWhatsAppSutReactionToTrigger(context, {
expectation: { anyEmoji: true },
timeoutMs: 30_000,
});
return `status reaction ${reaction.reaction?.emoji ?? "<unknown>"} observed`;
},
configMode: "allowlist",
expectReply: true,
input: `Reply with only this exact marker after normal processing: ${token}`,
matchText: token,
target: "dm",
};
},
};
export const whatsappQaStatusReactionLifecycleScenario: WhatsAppQaScenarioImplementation = {
posture: "user-path",
configOverrides: {
statusReactions: true,
},
buildRun: () => {
const token = `WHATSAPP_QA_STATUS_LIFECYCLE_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
afterReply: async (_reply, context) => {
const reactions = await waitForWhatsAppSutReactionSequenceToTrigger(context, {
emojis: ["👀", "✅"],
observedAfter: context.requestStartedAt,
timeoutMs: 60_000,
});
for (const reaction of reactions) {
context.recordObservedMessage(reaction);
}
return `status reaction lifecycle observed ${reactions
.map((reaction) => reaction.reaction?.emoji ?? "<unknown>")
.join(" -> ")}`;
},
configMode: "allowlist",
expectReply: true,
input: `Reply with only this exact marker after normal processing: ${token}`,
matchText: token,
target: "dm",
};
},
};
export const whatsappQaGroupAllowlistBlockScenario: WhatsAppQaScenarioImplementation = {
posture: "user-path",
configOverrides: {
blockGroupSender: true,
groupPolicy: "allowlist",
},
requiresGroupJid: true,
buildRun: () => {
const quietToken = `WHATSAPP_QA_GROUP_BLOCK_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
configMode: "allowlist",
expectReply: false,
input: `openclawqa blocked group should not reply with ${quietToken}`,
matchText: quietToken,
target: "group",
};
},
};
export const whatsappQaApprovalExecNativeScenario: WhatsAppQaScenarioImplementation = {
posture: "native-approval",
configOverrides: {
approvals: {
exec: true,
},
},
buildRun: () => ({
approvalKind: "exec",
decision: "allow-once",
kind: "approval",
token: `WHATSAPP_QA_EXEC_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`,
}),
};
export const whatsappQaApprovalExecReactionNativeScenario: WhatsAppQaScenarioImplementation = {
posture: "native-approval",
configOverrides: {
approvals: {
exec: true,
},
},
buildRun: () => ({
approvalKind: "exec",
decision: "allow-once",
decisionMode: "reaction",
kind: "approval",
token: `WHATSAPP_QA_EXEC_REACTION_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`,
}),
};
export const whatsappQaApprovalExecGroupReactionNativeScenario: WhatsAppQaScenarioImplementation = {
posture: "native-approval",
configOverrides: {
approvals: {
exec: true,
},
},
requiresGroupJid: true,
buildRun: () => ({
approvalKind: "exec",
decision: "allow-once",
decisionMode: "reaction",
kind: "approval",
target: "group",
token: `WHATSAPP_QA_GROUP_EXEC_REACTION_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`,
}),
};
export const whatsappQaApprovalPluginNativeScenario: WhatsAppQaScenarioImplementation = {
posture: "native-approval",
configOverrides: {
approvals: {
exec: true,
plugin: true,
},
},
buildRun: () => ({
approvalKind: "plugin",
decision: "allow-once",
kind: "approval",
token: `WHATSAPP_QA_PLUGIN_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`,
}),
};
extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenario-implementations.user-path.ts
+382
View File
@@ -0,0 +1,382 @@
// QA Lab WhatsApp user-path action and inbound media scenarios.
import { randomUUID } from "node:crypto";
import type { WhatsAppQaScenarioImplementation } from "./whatsapp-live.contracts.js";
import {
WHATSAPP_QA_AUDIO_OGG_OPUS_MIME,
WHATSAPP_QA_AUDIO_TRANSCRIPT_MARKER,
WHATSAPP_QA_ONE_PIXEL_PNG,
assertWhatsAppMessageFromSutPhone,
callWhatsAppGatewaySend,
createWhatsAppQaAudioOggOpusBuffer,
createWhatsAppQaAudioWavBuffer,
createWhatsAppQaPdfBuffer,
matchesWhatsAppSutReactionToTrigger,
waitForNoWhatsAppReply,
waitForScenarioObservedMessage,
waitForWhatsAppScenarioSutMessage,
waitForWhatsAppSutReactionToTrigger,
writeWhatsAppQaWorkspaceFixture,
} from "./whatsapp-live.operations.js";
export const whatsappQaAgentMessageActionReactScenario: WhatsAppQaScenarioImplementation = {
posture: "user-path",
configOverrides: {
actions: true,
},
buildRun: () => {
const token = `WHATSAPP_QA_AGENT_REACT_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
afterSend: async (context) => {
const reaction = await waitForWhatsAppSutReactionToTrigger(context, {
expectation: { emoji: "👍" },
timeoutMs: 60_000,
});
return `agent message reaction ${reaction.reaction?.emoji ?? "<unknown>"} observed`;
},
allowQuietWindowMessage: (message, context) =>
matchesWhatsAppSutReactionToTrigger(message, context, { emoji: "👍" }),
configMode: "allowlist",
expectReply: false,
input:
`React to this WhatsApp message with thumbs up for QA action check ${token}. ` +
"Do not send any visible text reply after the reaction.",
matchText: token,
quietWindowMs: 8_000,
target: "dm",
};
},
};
export const whatsappQaAgentMessageActionUploadFileScenario: WhatsAppQaScenarioImplementation = {
posture: "user-path",
configOverrides: {
actions: true,
},
buildRun: () => {
const token = `WHATSAPP_QA_AGENT_UPLOAD_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
afterSend: async (context) => {
const media = await waitForScenarioObservedMessage(context, {
observedAfter: context.requestStartedAt,
timeoutMs: 60_000,
match: (message) =>
message.kind === "media" &&
message.hasMedia === true &&
message.mediaType?.startsWith("image/") === true &&
message.text.includes(token),
});
return `agent upload-file media ${media.mediaType ?? "<unknown>"} observed`;
},
allowQuietWindowMessage: (message) =>
message.kind === "media" &&
message.mediaType?.startsWith("image/") === true &&
message.text.includes(token),
configMode: "allowlist",
expectReply: false,
input:
`Use the WhatsApp message tool upload-file action to send a PNG with caption ${token}. ` +
"Do not send any visible text reply after the upload.",
matchText: token,
quietWindowMs: 8_000,
target: "dm",
};
},
};
export const whatsappQaGroupAgentMessageActionReactScenario: WhatsAppQaScenarioImplementation = {
posture: "user-path",
configOverrides: {
actions: true,
},
requiresGroupJid: true,
buildRun: () => {
const token = `WHATSAPP_QA_GROUP_AGENT_REACT_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
afterSend: async (context) => {
const reaction = await waitForWhatsAppSutReactionToTrigger(context, {
expectation: { emoji: "👍" },
timeoutMs: 60_000,
});
return `group agent message reaction ${reaction.reaction?.emoji ?? "<unknown>"} observed`;
},
allowQuietWindowMessage: (message, context) =>
matchesWhatsAppSutReactionToTrigger(message, context, { emoji: "👍" }),
configMode: "allowlist",
expectReply: false,
input:
`openclawqa react to this WhatsApp group message with thumbs up for QA action check ${token}. ` +
"Do not send any visible text reply after the reaction.",
matchText: token,
quietWindowMs: 8_000,
target: "group",
};
},
};
export const whatsappQaGroupAgentMessageActionUploadFileScenario: WhatsAppQaScenarioImplementation =
{
posture: "user-path",
configOverrides: {
actions: true,
},
requiresGroupJid: true,
buildRun: () => {
const token = `WHATSAPP_QA_GROUP_AGENT_UPLOAD_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
afterSend: async (context) => {
const media = await waitForWhatsAppScenarioSutMessage(context, {
observedAfter: context.requestStartedAt,
targetKind: "group",
timeoutMs: 60_000,
match: (message) =>
message.kind === "media" &&
message.hasMedia === true &&
message.mediaType?.startsWith("image/") === true &&
message.text.includes(token),
});
return `group agent upload-file media ${media.mediaType ?? "<unknown>"} observed`;
},
allowQuietWindowMessage: (message) =>
message.kind === "media" &&
message.mediaType?.startsWith("image/") === true &&
message.text.includes(token),
configMode: "allowlist",
expectReply: false,
input:
`openclawqa use the WhatsApp message tool upload-file action to send a PNG with caption ${token}. ` +
"Do not send any visible text reply after the upload.",
matchText: token,
quietWindowMs: 8_000,
target: "group",
};
},
};
export const whatsappQaInboundReactionNoTriggerScenario: WhatsAppQaScenarioImplementation = {
posture: "user-path",
buildRun: () => {
const token = `WHATSAPP_QA_INBOUND_REACTION_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
afterReply: async (reply, context) => {
assertWhatsAppMessageFromSutPhone(reply, context);
if (!reply.messageId) {
throw new Error("WhatsApp SUT reply did not include a message id to react to.");
}
const reactionStartedAt = new Date();
await context.driver.sendReaction(context.target, reply.messageId, "❤️", {
fromMe: false,
});
await waitForNoWhatsAppReply({
driver: context.driver,
observedAfter: reactionStartedAt,
sutPhoneE164: context.sutPhoneE164,
target: "dm",
windowMs: 5_000,
});
return "driver reaction to SUT message did not trigger a fresh reply";
},
configMode: "allowlist",
expectReply: true,
input: `Reply with only this exact marker before inbound reaction check: ${token}`,
matchText: token,
target: "dm",
};
},
};
export const whatsappQaReplyContextIsolationScenario: WhatsAppQaScenarioImplementation = {
posture: "direct-gateway",
buildRun: () => {
const token = `WHATSAPP_QA_REPLY_ISOLATION_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
afterReply: async (_reply, context) => {
if (!context.sent.messageId) {
throw new Error("WhatsApp driver did not return a triggering message id.");
}
const quotedStartedAt = new Date();
await callWhatsAppGatewaySend(context, {
label: "quoted",
message: `${token}_QUOTED`,
replyToId: context.sent.messageId,
});
await waitForScenarioObservedMessage(context, {
observedAfter: quotedStartedAt,
diagnosticChecks: [
{
label: "textMarker",
match: (message) => message.text.includes(`${token}_QUOTED`),
},
{
label: "quotedMessageIdMatchesTrigger",
match: (message) => message.quoted?.messageId === context.sent.messageId,
},
],
match: (message) =>
message.text.includes(`${token}_QUOTED`) &&
message.quoted?.messageId === context.sent.messageId,
});
const freshStartedAt = new Date();
await callWhatsAppGatewaySend(context, {
label: "fresh",
message: `${token}_FRESH`,
});
const fresh = await waitForScenarioObservedMessage(context, {
observedAfter: freshStartedAt,
match: (message) => message.text.includes(`${token}_FRESH`),
});
if (fresh.quoted?.messageId) {
throw new Error(
`expected fresh WhatsApp send without quote metadata, got quoted message ${fresh.quoted.messageId}`,
);
}
return "quoted send and fresh send used independent reply context";
},
configMode: "allowlist",
expectReply: true,
input: `Reply with only this exact marker before reply isolation checks: ${token}`,
matchText: token,
target: "dm",
};
},
};
export const whatsappQaInboundImageCaptionScenario: WhatsAppQaScenarioImplementation = {
posture: "user-path",
buildRun: () => {
const token = `WHATSAPP_QA_IMAGE_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
configMode: "allowlist",
expectReply: true,
input: `This image caption asks you to reply with only this exact marker: ${token}`,
matchText: token,
sendMode: {
fileName: "whatsapp-qa.png",
kind: "media",
mediaBuffer: WHATSAPP_QA_ONE_PIXEL_PNG,
mediaType: "image/png",
},
target: "dm",
};
},
};
export const whatsappQaAudioPreflightScenario: WhatsAppQaScenarioImplementation = {
posture: "user-path",
configOverrides: {
audioPreflight: true,
},
buildRun: () => ({
configMode: "allowlist",
expectReply: true,
input: "",
matchText: WHATSAPP_QA_AUDIO_TRANSCRIPT_MARKER,
sendMode: {
fileName: "whatsapp-qa-audio.ogg",
kind: "media",
mediaBuffer: createWhatsAppQaAudioOggOpusBuffer(),
mediaType: WHATSAPP_QA_AUDIO_OGG_OPUS_MIME,
},
target: "dm",
}),
};
export const whatsappQaOutboundMediaMatrixScenario: WhatsAppQaScenarioImplementation = {
posture: "direct-gateway",
buildRun: () => {
const token = `WHATSAPP_QA_OUTBOUND_MEDIA_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
afterReply: async (_reply, context) => {
const mediaRootToken = randomUUID().slice(0, 8);
const imagePath = await writeWhatsAppQaWorkspaceFixture(context, {
buffer: WHATSAPP_QA_ONE_PIXEL_PNG,
fileName: `whatsapp-qa-${mediaRootToken}.png`,
});
const documentPath = await writeWhatsAppQaWorkspaceFixture(context, {
buffer: createWhatsAppQaPdfBuffer(),
fileName: `whatsapp-qa-${mediaRootToken}.pdf`,
});
const audioPath = await writeWhatsAppQaWorkspaceFixture(context, {
buffer: createWhatsAppQaAudioWavBuffer(),
fileName: `whatsapp-qa-${mediaRootToken}.wav`,
});
const imageStartedAt = new Date();
await callWhatsAppGatewaySend(context, {
label: "image",
mediaUrl: imagePath,
message: `${token}_IMAGE`,
});
await waitForScenarioObservedMessage(context, {
observedAfter: imageStartedAt,
match: (message) =>
message.kind === "media" &&
message.hasMedia === true &&
message.mediaType?.startsWith("image/") === true &&
message.text.includes(`${token}_IMAGE`),
});
const documentStartedAt = new Date();
await callWhatsAppGatewaySend(context, {
forceDocument: true,
label: "document",
mediaUrl: documentPath,
message: `${token}_DOCUMENT`,
});
await waitForScenarioObservedMessage(context, {
observedAfter: documentStartedAt,
match: (message) =>
message.kind === "media" &&
message.hasMedia === true &&
(message.mediaType === "application/pdf" ||
message.mediaFileName?.endsWith(".pdf") === true) &&
message.text.includes(`${token}_DOCUMENT`),
});
const audioStartedAt = new Date();
await callWhatsAppGatewaySend(context, {
asVoice: true,
label: "audio",
mediaUrl: audioPath,
message: `${token}_AUDIO`,
});
await waitForScenarioObservedMessage(context, {
observedAfter: audioStartedAt,
match: (message) =>
message.kind === "media" &&
message.hasMedia === true &&
message.mediaType?.startsWith("audio/") === true,
});
await waitForScenarioObservedMessage(context, {
observedAfter: audioStartedAt,
match: (message) => message.text.includes(`${token}_AUDIO`),
});
const multiStartedAt = new Date();
await callWhatsAppGatewaySend(context, {
label: "multi",
mediaUrls: [imagePath, documentPath],
message: `${token}_MULTI`,
});
await waitForScenarioObservedMessage(context, {
observedAfter: multiStartedAt,
match: (message) =>
message.kind === "media" && message.mediaType?.startsWith("image/") === true,
});
await waitForScenarioObservedMessage(context, {
observedAfter: multiStartedAt,
match: (message) =>
message.kind === "media" &&
(message.mediaType === "application/pdf" ||
message.mediaFileName?.endsWith(".pdf") === true),
});
return "gateway send delivered image, document, audio, and multi-media";
},
configMode: "allowlist",
expectReply: true,
input: `Reply with only this exact marker before outbound media checks: ${token}`,
matchText: token,
target: "dm",
};
},
};
extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenarios.capabilities.ts
-420
View File
@@ -1,420 +0,0 @@
// QA Lab WhatsApp Gateway capability and structured-message scenarios.
import { randomUUID } from "node:crypto";
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
import type { WhatsAppQaScenarioDefinition } from "./whatsapp-live.contracts.js";
import {
WHATSAPP_QA_AUDIO_OGG_OPUS_MIME,
WHATSAPP_QA_GROUP_AUDIO_TRANSCRIPT_MARKER,
WHATSAPP_QA_ONE_PIXEL_PNG,
callWhatsAppGatewayMessageAction,
callWhatsAppGatewayPoll,
callWhatsAppGatewaySend,
callWhatsAppGatewaySendConcurrently,
createWhatsAppQaAudioOggOpusBuffer,
createWhatsAppQaPdfBuffer,
requireWhatsAppTriggerMessageId,
runWhatsAppStructuredInboundChecks,
waitForScenarioObservedMessage,
waitForWhatsAppScenarioSutMessage,
waitForWhatsAppSutReactionToTrigger,
writeWhatsAppQaWorkspaceFixture,
} from "./whatsapp-live.operations.js";
export const WHATSAPP_QA_CAPABILITY_SCENARIOS: WhatsAppQaScenarioDefinition[] = [
{
id: "whatsapp-outbound-document-preserves-filename",
title: "WhatsApp direct Gateway document preserves filename and caption",
timeoutMs: 90_000,
buildRun: () => {
const token = `WHATSAPP_QA_DOCUMENT_FILE_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
afterReply: async (_reply, context) => {
const documentPath = await writeWhatsAppQaWorkspaceFixture(context, {
buffer: createWhatsAppQaPdfBuffer(),
fileName: `whatsapp-qa-report-${token}.pdf`,
});
const documentStartedAt = new Date();
await callWhatsAppGatewaySend(context, {
forceDocument: true,
label: "document-filename",
mediaUrl: documentPath,
message: `${token}_CAPTION`,
});
const document = await waitForScenarioObservedMessage(context, {
observedAfter: documentStartedAt,
match: (message) =>
message.kind === "media" &&
message.hasMedia === true &&
message.text.includes(`${token}_CAPTION`) &&
message.mediaFileName === `whatsapp-qa-report-${token}.pdf`,
});
return `document ${document.mediaFileName ?? "<missing filename>"} preserved`;
},
configMode: "allowlist",
expectReply: true,
input: `Reply with only this exact marker before document filename check: ${token}`,
matchText: token,
target: "dm",
};
},
},
{
id: "whatsapp-outbound-send-serialization",
title: "WhatsApp parallel Gateway sends deliver every outbound message",
timeoutMs: 90_000,
buildRun: () => {
const token = `WHATSAPP_QA_SERIAL_SEND_${randomUUID().slice(0, 8).toUpperCase()}`;
const markers = Array.from({ length: 5 }, (_, index) => `${token}_${index + 1}`);
return {
afterReply: async (_reply, context) => {
const sendsStartedAt = new Date();
await callWhatsAppGatewaySendConcurrently(
context,
markers.map((marker, index) => ({
label: `parallel-${index + 1}`,
message: marker,
})),
);
await Promise.all(
markers.map((marker) =>
waitForScenarioObservedMessage(context, {
observedAfter: sendsStartedAt,
match: (message) => message.kind === "text" && message.text.includes(marker),
}),
),
);
return `gateway parallel send delivered ${markers.length}/${markers.length} messages`;
},
configMode: "allowlist",
expectReply: true,
input: `Reply with only this exact marker before parallel send checks: ${token}`,
matchText: token,
target: "dm",
};
},
},
{
id: "whatsapp-outbound-poll",
title: "WhatsApp direct Gateway poll delivers outbound native poll",
timeoutMs: 90_000,
buildRun: () => {
const token = `WHATSAPP_QA_OUTBOUND_POLL_${randomUUID().slice(0, 8).toUpperCase()}`;
const question = `${token} choose one`;
return {
afterReply: async (_reply, context) => {
const pollStartedAt = new Date();
await callWhatsAppGatewayPoll(context, {
label: "poll",
options: ["alpha", "beta"],
question,
});
const poll = await waitForScenarioObservedMessage(context, {
observedAfter: pollStartedAt,
match: (message) =>
message.kind === "poll" &&
message.poll?.question === question &&
message.poll.options.includes("alpha") &&
message.poll.options.includes("beta"),
});
return `poll observed with ${poll.poll?.options.length ?? 0} options`;
},
configMode: "allowlist",
expectReply: true,
input: `Reply with only this exact marker before outbound poll check: ${token}`,
matchText: token,
target: "dm",
};
},
},
{
id: "whatsapp-group-outbound-media",
title: "WhatsApp direct Gateway send delivers media to a group",
timeoutMs: 120_000,
requiresGroupJid: true,
buildRun: () => {
const token = `WHATSAPP_QA_GROUP_OUTBOUND_MEDIA_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
afterReply: async (_reply, context) => {
const mediaRootToken = randomUUID().slice(0, 8);
const imagePath = await writeWhatsAppQaWorkspaceFixture(context, {
buffer: WHATSAPP_QA_ONE_PIXEL_PNG,
fileName: `whatsapp-qa-group-${mediaRootToken}.png`,
});
const documentPath = await writeWhatsAppQaWorkspaceFixture(context, {
buffer: createWhatsAppQaPdfBuffer(),
fileName: `whatsapp-qa-group-${mediaRootToken}.pdf`,
});
const imageStartedAt = new Date();
await callWhatsAppGatewaySend(context, {
label: "group-image",
mediaUrl: imagePath,
message: `${token}_IMAGE`,
});
await waitForWhatsAppScenarioSutMessage(context, {
observedAfter: imageStartedAt,
targetKind: "group",
match: (message) =>
message.kind === "media" &&
message.hasMedia === true &&
message.mediaType?.startsWith("image/") === true &&
message.text.includes(`${token}_IMAGE`),
});
const documentStartedAt = new Date();
await callWhatsAppGatewaySend(context, {
forceDocument: true,
label: "group-document",
mediaUrl: documentPath,
message: `${token}_DOCUMENT`,
});
await waitForWhatsAppScenarioSutMessage(context, {
observedAfter: documentStartedAt,
targetKind: "group",
match: (message) =>
message.kind === "media" &&
message.hasMedia === true &&
(message.mediaType === "application/pdf" ||
message.mediaFileName?.endsWith(".pdf") === true) &&
message.text.includes(`${token}_DOCUMENT`),
});
return "gateway send delivered image and document media to the group";
},
configMode: "allowlist",
expectReply: true,
input: `openclawqa reply with only this exact marker before group outbound media checks: ${token}`,
matchText: token,
target: "group",
};
},
},
{
id: "whatsapp-group-outbound-audio",
title: "WhatsApp direct Gateway send delivers audio to a group",
timeoutMs: 90_000,
requiresGroupJid: true,
buildRun: () => {
const token = `WHATSAPP_QA_GROUP_OUTBOUND_AUDIO_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
afterReply: async (_reply, context) => {
const audioPath = await writeWhatsAppQaWorkspaceFixture(context, {
buffer: createWhatsAppQaAudioOggOpusBuffer({ variant: "group-trigger" }),
fileName: `whatsapp-qa-group-audio-${token}.ogg`,
});
const audioStartedAt = new Date();
await callWhatsAppGatewaySend(context, {
asVoice: true,
label: "group-audio",
mediaUrl: audioPath,
message: `${token}_AUDIO`,
});
await waitForWhatsAppScenarioSutMessage(context, {
observedAfter: audioStartedAt,
targetKind: "group",
match: (message) =>
message.kind === "media" &&
message.hasMedia === true &&
message.mediaType?.startsWith("audio/") === true,
});
await waitForWhatsAppScenarioSutMessage(context, {
observedAfter: audioStartedAt,
targetKind: "group",
match: (message) => message.text.includes(`${token}_AUDIO`),
});
return "gateway send delivered audio media to the group";
},
configMode: "allowlist",
expectReply: true,
input: `openclawqa reply with only this exact marker before group outbound audio check: ${token}`,
matchText: token,
target: "group",
};
},
},
{
id: "whatsapp-group-outbound-poll",
title: "WhatsApp direct Gateway poll delivers native poll to a group",
timeoutMs: 90_000,
requiresGroupJid: true,
buildRun: () => {
const token = `WHATSAPP_QA_GROUP_OUTBOUND_POLL_${randomUUID().slice(0, 8).toUpperCase()}`;
const question = `${token} choose one`;
return {
afterReply: async (_reply, context) => {
const pollStartedAt = new Date();
await callWhatsAppGatewayPoll(context, {
label: "group-poll",
options: ["alpha", "beta"],
question,
});
const poll = await waitForWhatsAppScenarioSutMessage(context, {
observedAfter: pollStartedAt,
targetKind: "group",
match: (message) =>
message.kind === "poll" &&
message.poll?.question === question &&
message.poll.options.includes("alpha") &&
message.poll.options.includes("beta"),
});
return `group poll observed with ${poll.poll?.options.length ?? 0} options`;
},
configMode: "allowlist",
expectReply: true,
input: `openclawqa reply with only this exact marker before group outbound poll check: ${token}`,
matchText: token,
target: "group",
};
},
},
{
id: "whatsapp-message-actions",
title: "WhatsApp direct Gateway message.action react and upload-file execute",
timeoutMs: 120_000,
configOverrides: {
actions: true,
},
buildRun: () => {
const token = `WHATSAPP_QA_ACTIONS_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
afterReply: async (_reply, context) => {
const triggerMessageId = requireWhatsAppTriggerMessageId(context);
const reactionStartedAt = new Date();
await callWhatsAppGatewayMessageAction(context, {
action: "react",
label: "react",
params: {
emoji: "👍",
messageId: triggerMessageId,
},
});
await waitForWhatsAppSutReactionToTrigger(context, {
expectation: { emoji: "👍" },
observedAfter: reactionStartedAt,
});
const uploadStartedAt = new Date();
await callWhatsAppGatewayMessageAction(context, {
action: "upload-file",
label: "upload-file",
params: {
buffer: WHATSAPP_QA_ONE_PIXEL_PNG.toString("base64"),
caption: `${token}_UPLOAD`,
contentType: "image/png",
filename: "whatsapp-qa-upload.png",
},
});
await waitForScenarioObservedMessage(context, {
observedAfter: uploadStartedAt,
match: (message) =>
message.kind === "media" &&
message.mediaType?.startsWith("image/") === true &&
message.text.includes(`${token}_UPLOAD`),
});
return "message.action react and upload-file observed";
},
configMode: "allowlist",
expectReply: true,
input: `Reply with only this exact marker before action checks: ${token}`,
matchText: token,
target: "dm",
};
},
},
{
id: "whatsapp-inbound-structured-messages",
title: "WhatsApp inbound structured messages reach the agent",
timeoutMs: 240_000,
buildRun: () => {
const token = `WHATSAPP_QA_STRUCTURED_${randomUUID().slice(0, 8).toUpperCase()}`;
const locationToken = `${token}_LOCATION`;
const contactToken = `${token}_CONTACT`;
const stickerToken = `${token}_STICKER`;
const locationCoordinateText = "37.774900, -122.419400";
return {
afterReply: async (_reply, context) => {
const waitForStructuredReply = async (
label: string,
observedAfter: Date,
expectedToken: string,
) => {
try {
return await waitForScenarioObservedMessage(context, {
observedAfter,
timeoutMs: 60_000,
match: (message) => message.text.includes(expectedToken),
diagnosticChecks: [
{
label: "containsExpectedToken",
match: (message) => message.text.includes(expectedToken),
},
],
});
} catch (error) {
throw new Error(
`timed out waiting for WhatsApp structured ${label} reply (${expectedToken}): ${formatErrorMessage(error)}`,
{ cause: error },
);
}
};
await runWhatsAppStructuredInboundChecks({
contactToken,
documentToken: `${token}_DOCUMENT`,
driver: context.driver,
driverPhoneE164: context.driverPhoneE164,
locationToken,
stickerToken,
target: context.target,
waitForStructuredReply,
});
return "document, location, contact, and sticker elicited replies";
},
configMode: "allowlist",
expectReply: true,
input:
`When a later WhatsApp location message shows ${locationCoordinateText}, ` +
`reply with only this WhatsApp location marker: ${locationToken}. ` +
`When a later WhatsApp contact message appears, ` +
`reply with only this WhatsApp contact marker: ${contactToken}. ` +
`When a later WhatsApp sticker message appears, ` +
`reply with only this WhatsApp sticker marker: ${stickerToken}. ` +
`Reply with only this exact marker before structured inbound checks: ${token}`,
matchText: token,
target: "dm",
};
},
},
{
id: "whatsapp-group-audio-gating",
title: "WhatsApp group audio mention gating",
timeoutMs: 120_000,
configOverrides: {
audioPreflight: true,
},
requiredPluginIds: ["openai"],
requiresGroupJid: true,
buildRun: () => ({
configMode: "allowlist",
expectReply: true,
input: "",
matchText: WHATSAPP_QA_GROUP_AUDIO_TRANSCRIPT_MARKER,
quietInput: "",
quietSendMode: {
fileName: "whatsapp-qa-group-audio-quiet.ogg",
kind: "media",
mediaBuffer: createWhatsAppQaAudioOggOpusBuffer(),
mediaType: WHATSAPP_QA_AUDIO_OGG_OPUS_MIME,
},
quietWindowMs: 5_000,
sendMode: {
fileName: "whatsapp-qa-group-audio.ogg",
kind: "media",
mediaBuffer: createWhatsAppQaAudioOggOpusBuffer({
variant: "group-trigger",
}),
mediaType: WHATSAPP_QA_AUDIO_OGG_OPUS_MIME,
},
target: "group",
}),
},
];
extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenarios.conversation.ts
-379
View File
@@ -1,379 +0,0 @@
// QA Lab WhatsApp conversation and reply-context scenarios.
import { randomUUID } from "node:crypto";
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
import {
toWhatsAppQaError,
type WhatsAppQaMessageScenarioRun,
type WhatsAppQaScenarioDefinition,
} from "./whatsapp-live.contracts.js";
import {
assertWhatsAppMessageFromSutPhone,
assertWhatsAppMessagesFromSutPhone,
buildWhatsAppQuotedMessageKeyFromObservedMessage,
resolveWhatsAppQaNoReplyTarget,
waitForDistinctWhatsAppSutMessages,
waitForNoWhatsAppReply,
waitForWhatsAppScenarioSutMessage,
} from "./whatsapp-live.operations.js";
function buildWhatsAppQuoteReplyRun(target: "dm" | "group"): WhatsAppQaMessageScenarioRun {
const token = `WHATSAPP_QA_REPLY_TO_${target.toUpperCase()}_${randomUUID().slice(0, 8).toUpperCase()}`;
const input =
target === "group"
? `openclawqa reply with only this exact marker: ${token}`
: `Reply with only this exact marker: ${token}`;
return {
configMode: "allowlist",
expectReply: true,
input,
matchText: token,
target,
verify: (reply, context) => {
if (!context.sent.messageId) {
throw new Error("WhatsApp driver did not return a triggering message id.");
}
if (reply.quoted?.messageId !== context.sent.messageId) {
throw new Error(
`expected reply quote ${context.sent.messageId}, got ${reply.quoted?.messageId ?? "<missing>"}`,
);
}
},
};
}
export const WHATSAPP_QA_CONVERSATION_SCENARIOS: WhatsAppQaScenarioDefinition[] = [
{
id: "whatsapp-canary",
title: "WhatsApp DM canary",
timeoutMs: 60_000,
buildRun: () => {
const token = `WHATSAPP_QA_ECHO_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
configMode: "allowlist",
expectReply: true,
input: `Reply with only this exact marker: ${token}`,
matchText: token,
target: "dm",
};
},
},
{
id: "whatsapp-mention-gating",
title: "WhatsApp group mention gating",
timeoutMs: 60_000,
requiresGroupJid: true,
buildRun: () => {
const quietToken = `WHATSAPP_QA_GROUP_QUIET_${randomUUID().slice(0, 8).toUpperCase()}`;
const replyToken = `WHATSAPP_QA_GROUP_MENTION_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
configMode: "allowlist",
expectReply: true,
input: `openclawqa reply with only this exact marker: ${replyToken}`,
matchText: replyToken,
quietInput: `This group message is intentionally unmentioned. If you respond, include ${quietToken}.`,
quietMatchText: quietToken,
quietWindowMs: 5_000,
target: "group",
};
},
},
{
id: "whatsapp-group-pending-history-context",
title: "WhatsApp group pending history reaches mentioned turns",
timeoutMs: 90_000,
configOverrides: {
groupHistoryLimit: 50,
groupPolicy: "open",
inboundDebounceMs: 0,
replyToMode: "all",
},
requiresGroupJid: true,
buildRun: () => {
const suffix = randomUUID().slice(0, 8).toUpperCase();
const quietMarker = `WHATSAPP_QA_PENDING_HISTORY_QUIET_${suffix}`;
const contextSentinel = `WHATSAPP_QA_PENDING_HISTORY_CONTEXT_ONLY_${suffix}`;
const triggerMarker = `WHATSAPP_QA_PENDING_HISTORY_TRIGGER_${suffix}`;
const okMarker = `WHATSAPP_QA_PENDING_HISTORY_OK_${suffix}`;
return {
configMode: "open",
expectReply: true,
expectedSutMessageCount: 1,
input:
`openclawqa pending history context check ${triggerMarker}. ` +
`Reply with only ${okMarker} only if the previous quiet group message is present ` +
`in prior group context with its context-only sentinel. ` +
"Do not use current-message text as proof.",
matchText: okMarker,
quietInput: `quiet context marker ${quietMarker} ${contextSentinel}`,
quietWindowMs: 5_000,
target: "group",
};
},
},
{
id: "whatsapp-broadcast-group-fanout",
title: "WhatsApp group broadcast fans out to multiple agents",
timeoutMs: 120_000,
configOverrides: {
broadcast: {
agents: ["main", "qa-second"],
strategy: "sequential",
},
groupPolicy: "open",
},
requiresGroupJid: true,
buildRun: () => {
const token = `WHATSAPP_QA_BROADCAST_TOKEN_${randomUUID().slice(0, 8).toUpperCase()}`;
const mainMarker = `${token}_MAIN`;
const secondMarker = `${token}_SECOND`;
return {
afterReply: async (reply, context) => {
const replies = await waitForDistinctWhatsAppSutMessages(context, {
initialMessages: [reply],
matchers: [
(message) => message.text.includes(mainMarker),
(message) => message.text.includes(secondMarker),
],
observedAfter: context.requestStartedAt,
timeoutMs: 60_000,
});
assertWhatsAppMessagesFromSutPhone(replies, context);
return "broadcast fanout produced main and qa-second replies";
},
configMode: "open",
expectReply: true,
input: `openclawqa broadcast fanout check ${token}`,
matchText: mainMarker,
target: "group",
};
},
},
{
id: "whatsapp-group-activation-always",
title: "WhatsApp group activation always wakes unmentioned messages",
timeoutMs: 120_000,
configOverrides: {
groupPolicy: "open",
},
requiresGroupJid: true,
buildRun: () => {
const suffix = randomUUID().slice(0, 8).toUpperCase();
const alwaysMarker = `WHATSAPP_QA_ACTIVATION_ALWAYS_${suffix}`;
const quietMarker = `WHATSAPP_QA_ACTIVATION_QUIET_${suffix}`;
return {
afterReply: async (reply, context) => {
assertWhatsAppMessageFromSutPhone(reply, context);
let activationProbeError: unknown;
try {
const alwaysStartedAt = new Date();
await context.driver.sendText(
context.target,
`Group activation visible behavior marker ${alwaysMarker}`,
);
const alwaysReply = await waitForWhatsAppScenarioSutMessage(context, {
match: (message) => message.text.includes(alwaysMarker),
observedAfter: alwaysStartedAt,
targetKind: "group",
timeoutMs: 60_000,
});
assertWhatsAppMessageFromSutPhone(alwaysReply, context);
} catch (error) {
activationProbeError = error;
}
let restoreError: unknown;
const restoreStartedAt = new Date();
try {
await context.driver.sendText(context.target, "/activation mention");
const restoreReply = await waitForWhatsAppScenarioSutMessage(context, {
match: (message) => /\bactivation\b.*\bmention\b/iu.test(message.text),
observedAfter: restoreStartedAt,
targetKind: "group",
timeoutMs: 60_000,
});
assertWhatsAppMessageFromSutPhone(restoreReply, context);
} catch (error) {
restoreError = error;
}
if (activationProbeError && restoreError) {
throw new Error(
`activation always probe failed; additionally failed to restore mention mode: ${formatErrorMessage(restoreError)}`,
{ cause: activationProbeError },
);
}
if (activationProbeError) {
throw toWhatsAppQaError(activationProbeError);
}
if (restoreError) {
throw toWhatsAppQaError(restoreError);
}
const quietStartedAt = new Date();
await context.driver.sendText(
context.target,
`Group activation quiet marker ${quietMarker}`,
);
await waitForNoWhatsAppReply({
driver: context.driver,
observedAfter: quietStartedAt,
sutPhoneE164: context.sutPhoneE164,
windowMs: 5_000,
...resolveWhatsAppQaNoReplyTarget({
groupJid: context.target,
target: "group",
}),
});
return "activation always replied to an unmentioned group message and mention mode was restored";
},
configMode: "allowlist",
expectReply: true,
input: "/activation always",
matchText: /\bactivation\b.*\balways\b/iu,
target: "group",
};
},
},
{
id: "whatsapp-group-reply-to-bot-triggers",
title: "WhatsApp group reply to bot wakes without an explicit mention",
timeoutMs: 120_000,
configOverrides: {
groupPolicy: "open",
},
requiresGroupJid: true,
buildRun: () => {
const suffix = randomUUID().slice(0, 8).toUpperCase();
const seedMarker = `WHATSAPP_QA_REPLY_TO_BOT_SEED_${suffix}`;
const triggerMarker = `WHATSAPP_QA_REPLY_TO_BOT_TRIGGER_${suffix}`;
return {
afterReply: async (reply, context) => {
assertWhatsAppMessageFromSutPhone(reply, context);
const quotedStartedAt = new Date();
const quotedTrigger = await context.driver.sendText(
context.target,
`Quoted implicit reply trigger marker ${triggerMarker}`,
{
quotedMessageKey: buildWhatsAppQuotedMessageKeyFromObservedMessage(reply, {
remoteJid: context.target,
}),
},
);
if (!quotedTrigger.messageId) {
throw new Error("WhatsApp driver did not return a quoted trigger message id.");
}
const quotedTriggerMessageId = quotedTrigger.messageId;
const quotedReply = await waitForWhatsAppScenarioSutMessage(context, {
diagnosticChecks: [
{
label: "containsTriggerMarker",
match: (message) => message.text.includes(triggerMarker),
},
{
label: "quotesTrigger",
match: (message) => message.quoted?.messageId === quotedTriggerMessageId,
},
],
match: (message) =>
message.text.includes(triggerMarker) &&
message.quoted?.messageId === quotedTriggerMessageId,
observedAfter: quotedStartedAt,
targetKind: "group",
timeoutMs: 60_000,
});
assertWhatsAppMessageFromSutPhone(quotedReply, context);
return "quoted reply to bot triggered a group response without an explicit mention";
},
configMode: "allowlist",
expectReply: true,
input: `openclawqa Mentioned group seed marker ${seedMarker}`,
matchText: seedMarker,
target: "group",
};
},
},
{
id: "whatsapp-top-level-reply-shape",
title: "WhatsApp DM top-level reply shape",
timeoutMs: 60_000,
configOverrides: {
replyToMode: "off",
},
buildRun: () => {
const token = `WHATSAPP_QA_TOP_LEVEL_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
configMode: "allowlist",
expectReply: true,
input: `Reply with only this exact marker: ${token}`,
matchText: token,
target: "dm",
verify: (reply) => {
if (reply.quoted?.messageId) {
throw new Error(
`expected top-level WhatsApp reply without quote metadata, got quoted message ${reply.quoted.messageId}`,
);
}
},
};
},
},
{
id: "whatsapp-reply-to-message",
title: "WhatsApp DM reply-to mode quotes the triggering message",
timeoutMs: 60_000,
configOverrides: {
replyToMode: "all",
},
buildRun: () => buildWhatsAppQuoteReplyRun("dm"),
},
{
id: "whatsapp-group-reply-to-message",
title: "WhatsApp group reply-to mode quotes the triggering message",
timeoutMs: 60_000,
configOverrides: {
replyToMode: "all",
},
requiresGroupJid: true,
buildRun: () => buildWhatsAppQuoteReplyRun("group"),
},
{
id: "whatsapp-reply-to-mode-batched",
title: "WhatsApp batched reply-to mode quotes the queued message",
timeoutMs: 90_000,
configOverrides: {
inboundDebounceMs: 250,
replyToMode: "batched",
},
buildRun: () => {
const suffix = randomUUID().slice(0, 8).toUpperCase();
const firstToken = `WHATSAPP_QA_BATCHED_FIRST_${suffix}`;
const finalToken = `WHATSAPP_QA_BATCHED_FINAL_${suffix}`;
let secondMessageId: string | undefined;
return {
afterSend: async (context) => {
const second = await context.driver.sendText(
context.target,
`Second batched WhatsApp QA message. Reply with only this exact marker: ${finalToken} only if the previous queued message is visible in this same run context.`,
);
secondMessageId = second.messageId;
return "second batched message sent before debounce flush";
},
configMode: "allowlist",
expectReply: true,
input: `First batched WhatsApp QA message ${firstToken}. Wait for the next message before replying.`,
matchText: finalToken,
target: "dm",
verify: (reply) => {
if (!secondMessageId) {
throw new Error("WhatsApp driver did not return a second batched message id.");
}
if (reply.quoted?.messageId !== secondMessageId) {
throw new Error(
`expected batched reply quote ${secondMessageId}, got ${reply.quoted?.messageId ?? "<missing>"}`,
);
}
},
};
},
},
];
extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenarios.delivery.ts
-250
View File
@@ -1,250 +0,0 @@
// QA Lab WhatsApp delivery-shape, status, and approval scenarios.
import { randomUUID } from "node:crypto";
import type { WhatsAppQaScenarioDefinition } from "./whatsapp-live.contracts.js";
import {
callWhatsAppGatewaySend,
waitForScenarioObservedMessage,
waitForWhatsAppSutReactionSequenceToTrigger,
waitForWhatsAppSutReactionToTrigger,
} from "./whatsapp-live.operations.js";
export const WHATSAPP_QA_DELIVERY_SCENARIOS: WhatsAppQaScenarioDefinition[] = [
{
id: "whatsapp-reply-delivery-shape",
title: "WhatsApp direct Gateway send chunks long replies",
timeoutMs: 120_000,
buildRun: () => {
const token = `WHATSAPP_QA_REPLY_SHAPE_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
afterReply: async (_reply, context) => {
if (!context.sent.messageId) {
throw new Error("WhatsApp driver did not return a triggering message id.");
}
const quotedTriggerMessageId = context.sent.messageId;
const chunkStartedAt = new Date();
const longText = `${token}_LONG_BEGIN\n${"A".repeat(4_500)}\n${token}_LONG_END`;
await callWhatsAppGatewaySend(context, {
label: "long-reply",
message: longText,
replyToId: quotedTriggerMessageId,
});
const firstChunk = await waitForScenarioObservedMessage(context, {
observedAfter: chunkStartedAt,
diagnosticChecks: [
{
label: "longBeginMarker",
match: (message) => message.text.includes(`${token}_LONG_BEGIN`),
},
{
label: "quotesTrigger",
match: (message) => message.quoted?.messageId === quotedTriggerMessageId,
},
],
match: (message) =>
message.text.includes(`${token}_LONG_BEGIN`) &&
message.quoted?.messageId === quotedTriggerMessageId,
});
const secondChunk = await waitForScenarioObservedMessage(context, {
observedAfter: chunkStartedAt,
diagnosticChecks: [
{
label: "longEndMarker",
match: (message) => message.text.includes(`${token}_LONG_END`),
},
{
label: "quotesTrigger",
match: (message) => message.quoted?.messageId === quotedTriggerMessageId,
},
],
match: (message) =>
message.messageId !== firstChunk.messageId &&
message.text.includes(`${token}_LONG_END`) &&
message.quoted?.messageId === quotedTriggerMessageId,
});
return `long reply chunked across ${firstChunk.messageId ?? "<first>"} and ${secondChunk.messageId ?? "<second>"}`;
},
configMode: "allowlist",
expectReply: true,
input: `Reply with only this exact marker before reply-shape checks: ${token}`,
matchText: token,
target: "dm",
};
},
},
{
id: "whatsapp-stream-final-message-accounting",
title: "WhatsApp streamed final response has exactly the final chunks",
timeoutMs: 90_000,
buildRun: () => ({
configMode: "allowlist",
expectReply: true,
expectedJoinedSutTextIncludes: ["WHATSAPP-LONG-FINAL-BEGIN", "WHATSAPP-LONG-FINAL-END"],
expectedSutMessageCount: 2,
input: "WhatsApp long final QA check. Use the scripted long final response.",
matchText: "WHATSAPP-LONG-FINAL-BEGIN",
settleMs: 4_000,
target: "dm",
}),
},
{
id: "whatsapp-approval-exec-deny-native",
title: "WhatsApp native exec approval prompt denies",
timeoutMs: 60_000,
configOverrides: {
approvals: {
exec: true,
},
},
buildRun: () => ({
approvalKind: "exec",
decision: "deny",
kind: "approval",
token: `WHATSAPP_QA_EXEC_DENY_${randomUUID().slice(0, 8).toUpperCase()}`,
}),
},
{
id: "whatsapp-status-reactions",
title: "WhatsApp status reactions are observable",
timeoutMs: 60_000,
configOverrides: {
statusReactions: true,
},
buildRun: () => {
const token = `WHATSAPP_QA_STATUS_REACTION_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
afterSend: async (context) => {
const reaction = await waitForWhatsAppSutReactionToTrigger(context, {
expectation: { anyEmoji: true },
timeoutMs: 30_000,
});
return `status reaction ${reaction.reaction?.emoji ?? "<unknown>"} observed`;
},
configMode: "allowlist",
expectReply: true,
input: `Reply with only this exact marker after normal processing: ${token}`,
matchText: token,
target: "dm",
};
},
},
{
id: "whatsapp-status-reaction-lifecycle",
title: "WhatsApp status reaction lifecycle updates the triggering message",
timeoutMs: 90_000,
configOverrides: {
statusReactions: true,
},
buildRun: () => {
const token = `WHATSAPP_QA_STATUS_LIFECYCLE_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
afterReply: async (_reply, context) => {
const reactions = await waitForWhatsAppSutReactionSequenceToTrigger(context, {
emojis: ["👀", "✅"],
observedAfter: context.requestStartedAt,
timeoutMs: 60_000,
});
for (const reaction of reactions) {
context.recordObservedMessage(reaction);
}
return `status reaction lifecycle observed ${reactions
.map((reaction) => reaction.reaction?.emoji ?? "<unknown>")
.join(" -> ")}`;
},
configMode: "allowlist",
expectReply: true,
input: `Reply with only this exact marker after normal processing: ${token}`,
matchText: token,
target: "dm",
};
},
},
{
id: "whatsapp-group-allowlist-block",
title: "WhatsApp group outside allowlist stays quiet",
timeoutMs: 8_000,
configOverrides: {
blockGroupSender: true,
groupPolicy: "allowlist",
},
requiresGroupJid: true,
buildRun: () => {
const quietToken = `WHATSAPP_QA_GROUP_BLOCK_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
configMode: "allowlist",
expectReply: false,
input: `openclawqa blocked group should not reply with ${quietToken}`,
matchText: quietToken,
target: "group",
};
},
},
{
id: "whatsapp-approval-exec-native",
title: "WhatsApp native exec approval prompt resolves",
timeoutMs: 60_000,
configOverrides: {
approvals: {
exec: true,
},
},
buildRun: () => ({
approvalKind: "exec",
decision: "allow-once",
kind: "approval",
token: `WHATSAPP_QA_EXEC_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`,
}),
},
{
id: "whatsapp-approval-exec-reaction-native",
title: "WhatsApp native exec approval resolves from reaction",
timeoutMs: 60_000,
configOverrides: {
approvals: {
exec: true,
},
},
buildRun: () => ({
approvalKind: "exec",
decision: "allow-once",
decisionMode: "reaction",
kind: "approval",
token: `WHATSAPP_QA_EXEC_REACTION_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`,
}),
},
{
id: "whatsapp-approval-exec-group-reaction-native",
title: "WhatsApp group-origin exec approval resolves from reaction",
timeoutMs: 60_000,
configOverrides: {
approvals: {
exec: true,
},
},
requiresGroupJid: true,
buildRun: () => ({
approvalKind: "exec",
decision: "allow-once",
decisionMode: "reaction",
kind: "approval",
target: "group",
token: `WHATSAPP_QA_GROUP_EXEC_REACTION_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`,
}),
},
{
id: "whatsapp-approval-plugin-native",
title: "WhatsApp native plugin approval prompt resolves with exec approvals enabled",
timeoutMs: 60_000,
configOverrides: {
approvals: {
exec: true,
plugin: true,
},
},
buildRun: () => ({
approvalKind: "plugin",
decision: "allow-once",
kind: "approval",
token: `WHATSAPP_QA_PLUGIN_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`,
}),
},
];
extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenarios.ts
-21
View File
@@ -1,21 +0,0 @@
// QA Lab WhatsApp live scenario catalog and selection.
import type { WhatsAppQaScenarioDefinition } from "./whatsapp-live.contracts.js";
import { WHATSAPP_QA_CAPABILITY_SCENARIOS } from "./whatsapp-live.scenarios.capabilities.js";
import { WHATSAPP_QA_CONVERSATION_SCENARIOS } from "./whatsapp-live.scenarios.conversation.js";
import { WHATSAPP_QA_DELIVERY_SCENARIOS } from "./whatsapp-live.scenarios.delivery.js";
import { WHATSAPP_QA_USER_PATH_SCENARIOS } from "./whatsapp-live.scenarios.user-path.js";
const WHATSAPP_QA_SCENARIOS: WhatsAppQaScenarioDefinition[] = [
...WHATSAPP_QA_CONVERSATION_SCENARIOS,
...WHATSAPP_QA_USER_PATH_SCENARIOS,
...WHATSAPP_QA_CAPABILITY_SCENARIOS,
...WHATSAPP_QA_DELIVERY_SCENARIOS,
];
export function getWhatsAppQaScenarioDefinition(id: string) {
const scenario = WHATSAPP_QA_SCENARIOS.find((candidate) => candidate.id === id);
if (!scenario) {
throw new Error(`unknown WhatsApp QA scenario id: ${id}`);
}
return scenario;
}
extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenarios.user-path.ts
-394
View File
@@ -1,394 +0,0 @@
// QA Lab WhatsApp user-path action and inbound media scenarios.
import { randomUUID } from "node:crypto";
import type { WhatsAppQaScenarioDefinition } from "./whatsapp-live.contracts.js";
import {
WHATSAPP_QA_AUDIO_OGG_OPUS_MIME,
WHATSAPP_QA_AUDIO_TRANSCRIPT_MARKER,
WHATSAPP_QA_ONE_PIXEL_PNG,
assertWhatsAppMessageFromSutPhone,
callWhatsAppGatewaySend,
createWhatsAppQaAudioOggOpusBuffer,
createWhatsAppQaAudioWavBuffer,
createWhatsAppQaPdfBuffer,
matchesWhatsAppSutReactionToTrigger,
waitForNoWhatsAppReply,
waitForScenarioObservedMessage,
waitForWhatsAppScenarioSutMessage,
waitForWhatsAppSutReactionToTrigger,
writeWhatsAppQaWorkspaceFixture,
} from "./whatsapp-live.operations.js";
export const WHATSAPP_QA_USER_PATH_SCENARIOS: WhatsAppQaScenarioDefinition[] = [
{
id: "whatsapp-agent-message-action-react",
title: "WhatsApp user-path agent reaction uses the message tool",
timeoutMs: 90_000,
configOverrides: {
actions: true,
},
buildRun: () => {
const token = `WHATSAPP_QA_AGENT_REACT_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
afterSend: async (context) => {
const reaction = await waitForWhatsAppSutReactionToTrigger(context, {
expectation: { emoji: "👍" },
timeoutMs: 60_000,
});
return `agent message reaction ${reaction.reaction?.emoji ?? "<unknown>"} observed`;
},
allowQuietWindowMessage: (message, context) =>
matchesWhatsAppSutReactionToTrigger(message, context, { emoji: "👍" }),
configMode: "allowlist",
expectReply: false,
input:
`React to this WhatsApp message with thumbs up for QA action check ${token}. ` +
"Do not send any visible text reply after the reaction.",
matchText: token,
quietWindowMs: 8_000,
target: "dm",
};
},
},
{
id: "whatsapp-agent-message-action-upload-file",
title: "WhatsApp user-path agent upload-file sends media",
timeoutMs: 90_000,
configOverrides: {
actions: true,
},
buildRun: () => {
const token = `WHATSAPP_QA_AGENT_UPLOAD_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
afterSend: async (context) => {
const media = await waitForScenarioObservedMessage(context, {
observedAfter: context.requestStartedAt,
timeoutMs: 60_000,
match: (message) =>
message.kind === "media" &&
message.hasMedia === true &&
message.mediaType?.startsWith("image/") === true &&
message.text.includes(token),
});
return `agent upload-file media ${media.mediaType ?? "<unknown>"} observed`;
},
allowQuietWindowMessage: (message) =>
message.kind === "media" &&
message.mediaType?.startsWith("image/") === true &&
message.text.includes(token),
configMode: "allowlist",
expectReply: false,
input:
`Use the WhatsApp message tool upload-file action to send a PNG with caption ${token}. ` +
"Do not send any visible text reply after the upload.",
matchText: token,
quietWindowMs: 8_000,
target: "dm",
};
},
},
{
id: "whatsapp-group-agent-message-action-react",
title: "WhatsApp group user-path agent reaction uses the message tool",
timeoutMs: 90_000,
configOverrides: {
actions: true,
},
requiresGroupJid: true,
buildRun: () => {
const token = `WHATSAPP_QA_GROUP_AGENT_REACT_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
afterSend: async (context) => {
const reaction = await waitForWhatsAppSutReactionToTrigger(context, {
expectation: { emoji: "👍" },
timeoutMs: 60_000,
});
return `group agent message reaction ${reaction.reaction?.emoji ?? "<unknown>"} observed`;
},
allowQuietWindowMessage: (message, context) =>
matchesWhatsAppSutReactionToTrigger(message, context, { emoji: "👍" }),
configMode: "allowlist",
expectReply: false,
input:
`openclawqa react to this WhatsApp group message with thumbs up for QA action check ${token}. ` +
"Do not send any visible text reply after the reaction.",
matchText: token,
quietWindowMs: 8_000,
target: "group",
};
},
},
{
id: "whatsapp-group-agent-message-action-upload-file",
title: "WhatsApp group user-path agent upload-file sends media",
timeoutMs: 90_000,
configOverrides: {
actions: true,
},
requiresGroupJid: true,
buildRun: () => {
const token = `WHATSAPP_QA_GROUP_AGENT_UPLOAD_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
afterSend: async (context) => {
const media = await waitForWhatsAppScenarioSutMessage(context, {
observedAfter: context.requestStartedAt,
targetKind: "group",
timeoutMs: 60_000,
match: (message) =>
message.kind === "media" &&
message.hasMedia === true &&
message.mediaType?.startsWith("image/") === true &&
message.text.includes(token),
});
return `group agent upload-file media ${media.mediaType ?? "<unknown>"} observed`;
},
allowQuietWindowMessage: (message) =>
message.kind === "media" &&
message.mediaType?.startsWith("image/") === true &&
message.text.includes(token),
configMode: "allowlist",
expectReply: false,
input:
`openclawqa use the WhatsApp message tool upload-file action to send a PNG with caption ${token}. ` +
"Do not send any visible text reply after the upload.",
matchText: token,
quietWindowMs: 8_000,
target: "group",
};
},
},
{
id: "whatsapp-inbound-reaction-no-trigger",
title: "WhatsApp inbound user reaction does not start a fresh run",
timeoutMs: 90_000,
buildRun: () => {
const token = `WHATSAPP_QA_INBOUND_REACTION_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
afterReply: async (reply, context) => {
assertWhatsAppMessageFromSutPhone(reply, context);
if (!reply.messageId) {
throw new Error("WhatsApp SUT reply did not include a message id to react to.");
}
const reactionStartedAt = new Date();
await context.driver.sendReaction(context.target, reply.messageId, "❤️", {
fromMe: false,
});
await waitForNoWhatsAppReply({
driver: context.driver,
observedAfter: reactionStartedAt,
sutPhoneE164: context.sutPhoneE164,
target: "dm",
windowMs: 5_000,
});
return "driver reaction to SUT message did not trigger a fresh reply";
},
configMode: "allowlist",
expectReply: true,
input: `Reply with only this exact marker before inbound reaction check: ${token}`,
matchText: token,
target: "dm",
};
},
},
{
id: "whatsapp-reply-context-isolation",
title: "WhatsApp direct Gateway send does not reuse prior quote context",
timeoutMs: 120_000,
buildRun: () => {
const token = `WHATSAPP_QA_REPLY_ISOLATION_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
afterReply: async (_reply, context) => {
if (!context.sent.messageId) {
throw new Error("WhatsApp driver did not return a triggering message id.");
}
const quotedStartedAt = new Date();
await callWhatsAppGatewaySend(context, {
label: "quoted",
message: `${token}_QUOTED`,
replyToId: context.sent.messageId,
});
await waitForScenarioObservedMessage(context, {
observedAfter: quotedStartedAt,
diagnosticChecks: [
{
label: "textMarker",
match: (message) => message.text.includes(`${token}_QUOTED`),
},
{
label: "quotedMessageIdMatchesTrigger",
match: (message) => message.quoted?.messageId === context.sent.messageId,
},
],
match: (message) =>
message.text.includes(`${token}_QUOTED`) &&
message.quoted?.messageId === context.sent.messageId,
});
const freshStartedAt = new Date();
await callWhatsAppGatewaySend(context, {
label: "fresh",
message: `${token}_FRESH`,
});
const fresh = await waitForScenarioObservedMessage(context, {
observedAfter: freshStartedAt,
match: (message) => message.text.includes(`${token}_FRESH`),
});
if (fresh.quoted?.messageId) {
throw new Error(
`expected fresh WhatsApp send without quote metadata, got quoted message ${fresh.quoted.messageId}`,
);
}
return "quoted send and fresh send used independent reply context";
},
configMode: "allowlist",
expectReply: true,
input: `Reply with only this exact marker before reply isolation checks: ${token}`,
matchText: token,
target: "dm",
};
},
},
{
id: "whatsapp-inbound-image-caption",
title: "WhatsApp inbound image caption reaches the agent",
timeoutMs: 60_000,
buildRun: () => {
const token = `WHATSAPP_QA_IMAGE_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
configMode: "allowlist",
expectReply: true,
input: `This image caption asks you to reply with only this exact marker: ${token}`,
matchText: token,
sendMode: {
fileName: "whatsapp-qa.png",
kind: "media",
mediaBuffer: WHATSAPP_QA_ONE_PIXEL_PNG,
mediaType: "image/png",
},
target: "dm",
};
},
},
{
id: "whatsapp-audio-preflight",
title: "WhatsApp inbound audio preflight transcript reaches the agent",
timeoutMs: 90_000,
configOverrides: {
audioPreflight: true,
},
requiredPluginIds: ["openai"],
buildRun: () => ({
configMode: "allowlist",
expectReply: true,
input: "",
matchText: WHATSAPP_QA_AUDIO_TRANSCRIPT_MARKER,
sendMode: {
fileName: "whatsapp-qa-audio.ogg",
kind: "media",
mediaBuffer: createWhatsAppQaAudioOggOpusBuffer(),
mediaType: WHATSAPP_QA_AUDIO_OGG_OPUS_MIME,
},
target: "dm",
}),
},
{
id: "whatsapp-outbound-media-matrix",
title: "WhatsApp direct Gateway send delivers outbound media variants",
timeoutMs: 120_000,
buildRun: () => {
const token = `WHATSAPP_QA_OUTBOUND_MEDIA_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
afterReply: async (_reply, context) => {
const mediaRootToken = randomUUID().slice(0, 8);
const imagePath = await writeWhatsAppQaWorkspaceFixture(context, {
buffer: WHATSAPP_QA_ONE_PIXEL_PNG,
fileName: `whatsapp-qa-${mediaRootToken}.png`,
});
const documentPath = await writeWhatsAppQaWorkspaceFixture(context, {
buffer: createWhatsAppQaPdfBuffer(),
fileName: `whatsapp-qa-${mediaRootToken}.pdf`,
});
const audioPath = await writeWhatsAppQaWorkspaceFixture(context, {
buffer: createWhatsAppQaAudioWavBuffer(),
fileName: `whatsapp-qa-${mediaRootToken}.wav`,
});
const imageStartedAt = new Date();
await callWhatsAppGatewaySend(context, {
label: "image",
mediaUrl: imagePath,
message: `${token}_IMAGE`,
});
await waitForScenarioObservedMessage(context, {
observedAfter: imageStartedAt,
match: (message) =>
message.kind === "media" &&
message.hasMedia === true &&
message.mediaType?.startsWith("image/") === true &&
message.text.includes(`${token}_IMAGE`),
});
const documentStartedAt = new Date();
await callWhatsAppGatewaySend(context, {
forceDocument: true,
label: "document",
mediaUrl: documentPath,
message: `${token}_DOCUMENT`,
});
await waitForScenarioObservedMessage(context, {
observedAfter: documentStartedAt,
match: (message) =>
message.kind === "media" &&
message.hasMedia === true &&
(message.mediaType === "application/pdf" ||
message.mediaFileName?.endsWith(".pdf") === true) &&
message.text.includes(`${token}_DOCUMENT`),
});
const audioStartedAt = new Date();
await callWhatsAppGatewaySend(context, {
asVoice: true,
label: "audio",
mediaUrl: audioPath,
message: `${token}_AUDIO`,
});
await waitForScenarioObservedMessage(context, {
observedAfter: audioStartedAt,
match: (message) =>
message.kind === "media" &&
message.hasMedia === true &&
message.mediaType?.startsWith("audio/") === true,
});
await waitForScenarioObservedMessage(context, {
observedAfter: audioStartedAt,
match: (message) => message.text.includes(`${token}_AUDIO`),
});
const multiStartedAt = new Date();
await callWhatsAppGatewaySend(context, {
label: "multi",
mediaUrls: [imagePath, documentPath],
message: `${token}_MULTI`,
});
await waitForScenarioObservedMessage(context, {
observedAfter: multiStartedAt,
match: (message) =>
message.kind === "media" && message.mediaType?.startsWith("image/") === true,
});
await waitForScenarioObservedMessage(context, {
observedAfter: multiStartedAt,
match: (message) =>
message.kind === "media" &&
(message.mediaType === "application/pdf" ||
message.mediaFileName?.endsWith(".pdf") === true),
});
return "gateway send delivered image, document, audio, and multi-media";
},
configMode: "allowlist",
expectReply: true,
input: `Reply with only this exact marker before outbound media checks: ${token}`,
matchText: token,
target: "dm",
};
},
},
];
extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.ts
+4 -7
View File
@@ -8,7 +8,7 @@ import {
acquireQaCredentialLease,
startQaCredentialLeaseHeartbeat,
} from "../live-transports/shared/credential-lease.runtime.js";
import { listSlackQaScenarioCatalog } from "../live-transports/slack/slack-live.scenarios.js";
import { resolveSlackQaScenarioIds } from "../live-transports/slack/scenario-selection.js";
import { isTruthyOptIn, trimToValue } from "../mantis-options.runtime.js";
import { createPhaseTimer, type MantisPhaseTimings } from "../mantis-phase-timer.runtime.js";
import {
@@ -214,12 +214,9 @@ function resolveScenarioIds(params: {
].join(", ")}. Unsupported: ${unsupported.join(", ")}.`,
);
}
const requested = new Set(scenarioIds);
// Slack selects scenarios from catalog order, not CLI order. The watcher
// must mirror that order or both sides can block on different checkpoints.
return listSlackQaScenarioCatalog()
.map((scenario) => scenario.id)
.filter((scenarioId) => requested.has(scenarioId));
// Mirror the YAML catalog order used by the Slack runner so the watcher
// and runner cannot block on different approval checkpoints.
return resolveSlackQaScenarioIds({ scenarioIds });
}
return scenarioIds;
}
extensions/qa-lab/src/scenario-module-flow.test.ts
+42
View File
@@ -0,0 +1,42 @@
import { describe, expect, it } from "vitest";
import { qaScenarioModuleFlow } from "./scenario-module-flow.js";
describe("QA scenario module flow", () => {
it("resolves a module export argument against the loaded scenario module", () => {
const flow = qaScenarioModuleFlow.moduleSchema.parse({
module: "./scenario-runtime.js",
call: "runScenario",
args: [{ expr: "scenarioContext" }, { moduleExport: "scenarioImplementation" }],
});
expect(qaScenarioModuleFlow.resolveFlow(flow, "Scenario title")).toMatchObject({
steps: [
{
actions: [
{
set: "scenarioModule",
value: { expr: 'await qaImport("./scenario-runtime.js")' },
},
{
args: [
{ expr: "scenarioContext" },
{ expr: 'scenarioModule["scenarioImplementation"]' },
],
call: "scenarioModule.runScenario",
},
],
},
],
});
});
it("rejects malformed module export arguments", () => {
expect(() =>
qaScenarioModuleFlow.moduleSchema.parse({
module: "./scenario-runtime.js",
call: "runScenario",
args: [{ moduleExport: "" }],
}),
).toThrow("moduleExport arguments require a non-empty string export name");
});
});
extensions/qa-lab/src/scenario-module-flow.ts
+31 -2
View File
@@ -1,10 +1,29 @@
// QA Lab scenario module references normalize into the canonical flow shape.
import { z } from "zod";
const qaFlowModuleExportArgSchema = z
.object({
moduleExport: z.string().trim().min(1),
})
.strict();
const qaFlowModuleArgSchema = z.unknown().superRefine((arg, ctx) => {
if (
typeof arg !== "object" ||
arg === null ||
!("moduleExport" in arg) ||
qaFlowModuleExportArgSchema.safeParse(arg).success
) {
return;
}
ctx.addIssue({
code: "custom",
message: "moduleExport arguments require a non-empty string export name",
});
});
const qaFlowModuleSchema = z.object({
module: z.string().trim().min(1),
call: z.string().trim().min(1),
args: z.array(z.unknown()).optional(),
args: z.array(qaFlowModuleArgSchema).optional(),
});
const qaFlowExecutionShape = {
providerMode: z.enum(["aimock", "live-frontier", "mock-openai"]).optional(),
@@ -37,6 +56,16 @@ function normalizeQaScenarioFileMetadata<
};
}
function resolveQaScenarioModuleArg(arg: unknown) {
const parsed = qaFlowModuleExportArgSchema.safeParse(arg);
if (!parsed.success) {
return arg;
}
return {
expr: `scenarioModule[${JSON.stringify(parsed.data.moduleExport)}]`,
};
}
function resolveQaScenarioFileFlow<TFlow extends QaScenarioFlowShape>(
flow: TFlow | QaScenarioModuleFlow | undefined,
title: string,
@@ -55,7 +84,7 @@ function resolveQaScenarioFileFlow<TFlow extends QaScenarioFlowShape>(
},
{
call: `scenarioModule.${flow.call}`,
...(flow.args ? { args: flow.args } : {}),
...(flow.args ? { args: flow.args.map(resolveQaScenarioModuleArg) } : {}),
saveAs: "result",
},
],
extensions/qa-lab/src/suite-runtime-flow.ts
+2
View File
@@ -340,6 +340,8 @@ export function createQaSuiteScenarioStepRunner(
gateway: env.gateway,
outputDir: env.outputDir,
primaryModel: env.primaryModel,
scenarioId: scenario.id,
scenarioTitle: scenario.title,
timeoutMs: execution.timeoutMs ?? deps.liveTurnTimeoutMs(env, 60_000),
waitForConfigRestartSettle: async (options) =>
await waitForConfigRestartSettle(env, options?.restartDelayMs, options?.timeoutMs),
extensions/qa-lab/src/suite.test.ts
+2
View File
@@ -165,6 +165,8 @@ describe("qa suite", () => {
config: { expected: "value" },
gateway: env.gateway,
outputDir: "/tmp/qa-output",
scenarioId: "matrix-preparation-failure",
scenarioTitle: "matrix-preparation-failure",
timeoutMs: 45_000,
waitForConfigRestartSettle: expect.any(Function),
});
qa/scenarios/channels/discord-canary.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 45000
retryCount: 1
suiteIsolation: isolated
config: { discordScenarioId: discord-canary }
flow:
module: ./live-transports/discord/scenario-runtime.js
call: runDiscordCanaryScenario
args: [{ expr: discordScenarioContext }]
call: runDiscordScenario
args:
- { expr: discordScenarioContext }
- { moduleExport: discordQaCanaryScenario }
qa/scenarios/channels/discord-mention-gating.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 8000
retryCount: 1
suiteIsolation: isolated
config: { discordScenarioId: discord-mention-gating }
flow:
module: ./live-transports/discord/scenario-runtime.js
call: runDiscordMentionGatingScenario
args: [{ expr: discordScenarioContext }]
call: runDiscordScenario
args:
- { expr: discordScenarioContext }
- { moduleExport: discordQaMentionGatingScenario }
qa/scenarios/channels/discord-native-help-command-registration.yaml
+4 -3
View File
@@ -10,8 +10,9 @@ scenario:
timeoutMs: 45000
retryCount: 1
suiteIsolation: isolated
config: { discordScenarioId: discord-native-help-command-registration }
flow:
module: ./live-transports/discord/scenario-runtime.js
call: runDiscordNativeHelpCommandRegistrationScenario
args: [{ expr: discordScenarioContext }]
call: runDiscordScenario
args:
- { expr: discordScenarioContext }
- { moduleExport: discordQaNativeHelpCommandRegistrationScenario }
qa/scenarios/channels/discord-status-reactions-tool-only.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 75000
retryCount: 1
suiteIsolation: isolated
config: { discordScenarioId: discord-status-reactions-tool-only }
flow:
module: ./live-transports/discord/scenario-runtime.js
call: runDiscordStatusReactionsToolOnlyScenario
args: [{ expr: discordScenarioContext }]
call: runDiscordScenario
args:
- { expr: discordScenarioContext }
- { moduleExport: discordQaStatusReactionsToolOnlyScenario }
qa/scenarios/channels/discord-thread-reply-filepath-attachment.yaml
+4 -3
View File
@@ -12,8 +12,9 @@ scenario:
timeoutMs: 45000
retryCount: 1
suiteIsolation: isolated
config: { discordScenarioId: discord-thread-reply-filepath-attachment }
flow:
module: ./live-transports/discord/scenario-runtime.js
call: runDiscordThreadReplyFilepathAttachmentScenario
args: [{ expr: discordScenarioContext }]
call: runDiscordScenario
args:
- { expr: discordScenarioContext }
- { moduleExport: discordQaThreadReplyFilepathAttachmentScenario }
qa/scenarios/channels/discord-voice-autojoin.yaml
+4 -3
View File
@@ -10,8 +10,9 @@ scenario:
timeoutMs: 60000
retryCount: 1
suiteIsolation: isolated
config: { discordScenarioId: discord-voice-autojoin }
flow:
module: ./live-transports/discord/scenario-runtime.js
call: runDiscordVoiceAutojoinScenario
args: [{ expr: discordScenarioContext }]
call: runDiscordScenario
args:
- { expr: discordScenarioContext }
- { moduleExport: discordQaVoiceAutojoinScenario }
qa/scenarios/channels/slack-allowlist-block.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 8000
retryCount: 1
suiteIsolation: isolated
config: { slackScenarioId: slack-allowlist-block }
flow:
module: ./live-transports/slack/scenario-runtime.js
call: runSlackAllowlistBlockScenario
args: [{ expr: slackScenarioContext }]
call: runSlackScenario
args:
- { expr: slackScenarioContext }
- { moduleExport: slackQaAllowlistBlockScenario }
qa/scenarios/channels/slack-approval-exec-native.yaml
+4 -3
View File
@@ -13,8 +13,9 @@ scenario:
timeoutMs: 60000
retryCount: 1
suiteIsolation: isolated
config: { slackScenarioId: slack-approval-exec-native }
flow:
module: ./live-transports/slack/scenario-runtime.js
call: runSlackApprovalExecNativeScenario
args: [{ expr: slackScenarioContext }]
call: runSlackScenario
args:
- { expr: slackScenarioContext }
- { moduleExport: slackQaApprovalExecNativeScenario }
qa/scenarios/channels/slack-approval-plugin-native.yaml
+4 -3
View File
@@ -13,8 +13,9 @@ scenario:
timeoutMs: 60000
retryCount: 1
suiteIsolation: isolated
config: { slackScenarioId: slack-approval-plugin-native }
flow:
module: ./live-transports/slack/scenario-runtime.js
call: runSlackApprovalPluginNativeScenario
args: [{ expr: slackScenarioContext }]
call: runSlackScenario
args:
- { expr: slackScenarioContext }
- { moduleExport: slackQaApprovalPluginNativeScenario }
qa/scenarios/channels/slack-canary.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 45000
retryCount: 1
suiteIsolation: isolated
config: { slackScenarioId: slack-canary }
flow:
module: ./live-transports/slack/scenario-runtime.js
call: runSlackCanaryScenario
args: [{ expr: slackScenarioContext }]
call: runSlackScenario
args:
- { expr: slackScenarioContext }
- { moduleExport: slackQaCanaryScenario }
qa/scenarios/channels/slack-channel-disabled-warning.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 8000
retryCount: 1
suiteIsolation: isolated
config: { slackScenarioId: slack-channel-disabled-warning }
flow:
module: ./live-transports/slack/scenario-runtime.js
call: runSlackChannelDisabledWarningScenario
args: [{ expr: slackScenarioContext }]
call: runSlackScenario
args:
- { expr: slackScenarioContext }
- { moduleExport: slackQaChannelDisabledWarningScenario }
qa/scenarios/channels/slack-chart-presentation-native.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 90000
retryCount: 1
suiteIsolation: isolated
config: { slackScenarioId: slack-chart-presentation-native }
flow:
module: ./live-transports/slack/scenario-runtime.js
call: runSlackChartPresentationNativeScenario
args: [{ expr: slackScenarioContext }]
call: runSlackScenario
args:
- { expr: slackScenarioContext }
- { moduleExport: slackQaChartPresentationNativeScenario }
qa/scenarios/channels/slack-codex-approval-exec-native.yaml
+4 -3
View File
@@ -14,8 +14,9 @@ scenario:
timeoutMs: 180000
retryCount: 1
suiteIsolation: isolated
config: { slackScenarioId: slack-codex-approval-exec-native }
flow:
module: ./live-transports/slack/scenario-runtime.js
call: runSlackCodexApprovalExecNativeScenario
args: [{ expr: slackScenarioContext }]
call: runSlackScenario
args:
- { expr: slackScenarioContext }
- { moduleExport: slackQaCodexApprovalExecNativeScenario }
qa/scenarios/channels/slack-codex-approval-plugin-native.yaml
+4 -3
View File
@@ -14,8 +14,9 @@ scenario:
timeoutMs: 180000
retryCount: 1
suiteIsolation: isolated
config: { slackScenarioId: slack-codex-approval-plugin-native }
flow:
module: ./live-transports/slack/scenario-runtime.js
call: runSlackCodexApprovalPluginNativeScenario
args: [{ expr: slackScenarioContext }]
call: runSlackScenario
args:
- { expr: slackScenarioContext }
- { moduleExport: slackQaCodexApprovalPluginNativeScenario }
qa/scenarios/channels/slack-mention-gating.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 8000
retryCount: 1
suiteIsolation: isolated
config: { slackScenarioId: slack-mention-gating }
flow:
module: ./live-transports/slack/scenario-runtime.js
call: runSlackMentionGatingScenario
args: [{ expr: slackScenarioContext }]
call: runSlackScenario
args:
- { expr: slackScenarioContext }
- { moduleExport: slackQaMentionGatingScenario }
qa/scenarios/channels/slack-mpim-app-mention-dedupe.yaml
+4 -3
View File
@@ -13,8 +13,9 @@ scenario:
timeoutMs: 180000
retryCount: 0
suiteIsolation: isolated
config: { slackScenarioId: slack-mpim-app-mention-dedupe }
flow:
module: ./live-transports/slack/scenario-runtime.js
call: runSlackMpimAppMentionDedupeScenario
args: [{ expr: slackScenarioContext }]
call: runSlackScenario
args:
- { expr: slackScenarioContext }
- { moduleExport: slackQaMpimAppMentionDedupeScenario }
qa/scenarios/channels/slack-progress-commentary-false.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 90000
retryCount: 1
suiteIsolation: isolated
config: { slackScenarioId: slack-progress-commentary-false }
flow:
module: ./live-transports/slack/scenario-runtime.js
call: runSlackProgressCommentaryFalseScenario
args: [{ expr: slackScenarioContext }]
call: runSlackScenario
args:
- { expr: slackScenarioContext }
- { moduleExport: slackQaProgressCommentaryFalseScenario }
qa/scenarios/channels/slack-progress-commentary-omitted.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 90000
retryCount: 1
suiteIsolation: isolated
config: { slackScenarioId: slack-progress-commentary-omitted }
flow:
module: ./live-transports/slack/scenario-runtime.js
call: runSlackProgressCommentaryOmittedScenario
args: [{ expr: slackScenarioContext }]
call: runSlackScenario
args:
- { expr: slackScenarioContext }
- { moduleExport: slackQaProgressCommentaryOmittedScenario }
qa/scenarios/channels/slack-progress-commentary-true.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 90000
retryCount: 1
suiteIsolation: isolated
config: { slackScenarioId: slack-progress-commentary-true }
flow:
module: ./live-transports/slack/scenario-runtime.js
call: runSlackProgressCommentaryTrueScenario
args: [{ expr: slackScenarioContext }]
call: runSlackScenario
args:
- { expr: slackScenarioContext }
- { moduleExport: slackQaProgressCommentaryTrueScenario }
qa/scenarios/channels/slack-progress-commentary-verbose-dedupe.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 90000
retryCount: 1
suiteIsolation: isolated
config: { slackScenarioId: slack-progress-commentary-verbose-dedupe }
flow:
module: ./live-transports/slack/scenario-runtime.js
call: runSlackProgressCommentaryVerboseDedupeScenario
args: [{ expr: slackScenarioContext }]
call: runSlackScenario
args:
- { expr: slackScenarioContext }
- { moduleExport: slackQaProgressCommentaryVerboseDedupeScenario }
qa/scenarios/channels/slack-reaction-glyph-native.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 90000
retryCount: 1
suiteIsolation: isolated
config: { slackScenarioId: slack-reaction-glyph-native }
flow:
module: ./live-transports/slack/scenario-runtime.js
call: runSlackReactionGlyphNativeScenario
args: [{ expr: slackScenarioContext }]
call: runSlackScenario
args:
- { expr: slackScenarioContext }
- { moduleExport: slackQaReactionGlyphNativeScenario }
qa/scenarios/channels/slack-table-invalid-blocks-fallback.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 45000
retryCount: 1
suiteIsolation: isolated
config: { slackScenarioId: slack-table-invalid-blocks-fallback }
flow:
module: ./live-transports/slack/scenario-runtime.js
call: runSlackTableInvalidBlocksFallbackScenario
args: [{ expr: slackScenarioContext }]
call: runSlackScenario
args:
- { expr: slackScenarioContext }
- { moduleExport: slackQaTableInvalidBlocksFallbackScenario }
qa/scenarios/channels/slack-table-presentation-native.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 90000
retryCount: 1
suiteIsolation: isolated
config: { slackScenarioId: slack-table-presentation-native }
flow:
module: ./live-transports/slack/scenario-runtime.js
call: runSlackTablePresentationNativeScenario
args: [{ expr: slackScenarioContext }]
call: runSlackScenario
args:
- { expr: slackScenarioContext }
- { moduleExport: slackQaTablePresentationNativeScenario }
qa/scenarios/channels/slack-top-level-reply-shape.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 45000
retryCount: 1
suiteIsolation: isolated
config: { slackScenarioId: slack-top-level-reply-shape }
flow:
module: ./live-transports/slack/scenario-runtime.js
call: runSlackTopLevelReplyShapeScenario
args: [{ expr: slackScenarioContext }]
call: runSlackScenario
args:
- { expr: slackScenarioContext }
- { moduleExport: slackQaTopLevelReplyShapeScenario }
qa/scenarios/channels/whatsapp-agent-message-action-react.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 90000
retryCount: 1
suiteIsolation: isolated
config: { whatsappScenarioId: whatsapp-agent-message-action-react }
flow:
module: ./live-transports/whatsapp/scenario-runtime.js
call: runWhatsAppAgentMessageActionReactScenario
args: [{ expr: whatsappScenarioContext }]
call: runWhatsAppScenario
args:
- { expr: whatsappScenarioContext }
- { moduleExport: whatsappQaAgentMessageActionReactScenario }
qa/scenarios/channels/whatsapp-agent-message-action-upload-file.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 90000
retryCount: 1
suiteIsolation: isolated
config: { whatsappScenarioId: whatsapp-agent-message-action-upload-file }
flow:
module: ./live-transports/whatsapp/scenario-runtime.js
call: runWhatsAppAgentMessageActionUploadFileScenario
args: [{ expr: whatsappScenarioContext }]
call: runWhatsAppScenario
args:
- { expr: whatsappScenarioContext }
- { moduleExport: whatsappQaAgentMessageActionUploadFileScenario }
qa/scenarios/channels/whatsapp-approval-exec-deny-native.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 60000
retryCount: 1
suiteIsolation: isolated
config: { whatsappScenarioId: whatsapp-approval-exec-deny-native }
flow:
module: ./live-transports/whatsapp/scenario-runtime.js
call: runWhatsAppApprovalExecDenyNativeScenario
args: [{ expr: whatsappScenarioContext }]
call: runWhatsAppScenario
args:
- { expr: whatsappScenarioContext }
- { moduleExport: whatsappQaApprovalExecDenyNativeScenario }
qa/scenarios/channels/whatsapp-approval-exec-group-reaction-native.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 60000
retryCount: 1
suiteIsolation: isolated
config: { whatsappScenarioId: whatsapp-approval-exec-group-reaction-native }
flow:
module: ./live-transports/whatsapp/scenario-runtime.js
call: runWhatsAppApprovalExecGroupReactionNativeScenario
args: [{ expr: whatsappScenarioContext }]
call: runWhatsAppScenario
args:
- { expr: whatsappScenarioContext }
- { moduleExport: whatsappQaApprovalExecGroupReactionNativeScenario }
qa/scenarios/channels/whatsapp-approval-exec-native.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 60000
retryCount: 1
suiteIsolation: isolated
config: { whatsappScenarioId: whatsapp-approval-exec-native }
flow:
module: ./live-transports/whatsapp/scenario-runtime.js
call: runWhatsAppApprovalExecNativeScenario
args: [{ expr: whatsappScenarioContext }]
call: runWhatsAppScenario
args:
- { expr: whatsappScenarioContext }
- { moduleExport: whatsappQaApprovalExecNativeScenario }
qa/scenarios/channels/whatsapp-approval-exec-reaction-native.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 60000
retryCount: 1
suiteIsolation: isolated
config: { whatsappScenarioId: whatsapp-approval-exec-reaction-native }
flow:
module: ./live-transports/whatsapp/scenario-runtime.js
call: runWhatsAppApprovalExecReactionNativeScenario
args: [{ expr: whatsappScenarioContext }]
call: runWhatsAppScenario
args:
- { expr: whatsappScenarioContext }
- { moduleExport: whatsappQaApprovalExecReactionNativeScenario }
qa/scenarios/channels/whatsapp-approval-plugin-native.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 60000
retryCount: 1
suiteIsolation: isolated
config: { whatsappScenarioId: whatsapp-approval-plugin-native }
flow:
module: ./live-transports/whatsapp/scenario-runtime.js
call: runWhatsAppApprovalPluginNativeScenario
args: [{ expr: whatsappScenarioContext }]
call: runWhatsAppScenario
args:
- { expr: whatsappScenarioContext }
- { moduleExport: whatsappQaApprovalPluginNativeScenario }
qa/scenarios/channels/whatsapp-audio-preflight.yaml
+6 -3
View File
@@ -2,6 +2,8 @@ title: WhatsApp inbound audio preflight transcript reaches the agent
scenario:
id: whatsapp-audio-preflight
surface: channels
plugins:
- openai
coverage:
primary:
- whatsapp.inbound-media-download
@@ -11,8 +13,9 @@ scenario:
timeoutMs: 90000
retryCount: 1
suiteIsolation: isolated
config: { whatsappScenarioId: whatsapp-audio-preflight }
flow:
module: ./live-transports/whatsapp/scenario-runtime.js
call: runWhatsAppAudioPreflightScenario
args: [{ expr: whatsappScenarioContext }]
call: runWhatsAppScenario
args:
- { expr: whatsappScenarioContext }
- { moduleExport: whatsappQaAudioPreflightScenario }
qa/scenarios/channels/whatsapp-broadcast-group-fanout.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 120000
retryCount: 1
suiteIsolation: isolated
config: { whatsappScenarioId: whatsapp-broadcast-group-fanout }
flow:
module: ./live-transports/whatsapp/scenario-runtime.js
call: runWhatsAppBroadcastGroupFanoutScenario
args: [{ expr: whatsappScenarioContext }]
call: runWhatsAppScenario
args:
- { expr: whatsappScenarioContext }
- { moduleExport: whatsappQaBroadcastGroupFanoutScenario }
qa/scenarios/channels/whatsapp-canary.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 60000
retryCount: 1
suiteIsolation: isolated
config: { whatsappScenarioId: whatsapp-canary }
flow:
module: ./live-transports/whatsapp/scenario-runtime.js
call: runWhatsAppCanaryScenario
args: [{ expr: whatsappScenarioContext }]
call: runWhatsAppScenario
args:
- { expr: whatsappScenarioContext }
- { moduleExport: whatsappQaCanaryScenario }
qa/scenarios/channels/whatsapp-group-activation-always.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 120000
retryCount: 1
suiteIsolation: isolated
config: { whatsappScenarioId: whatsapp-group-activation-always }
flow:
module: ./live-transports/whatsapp/scenario-runtime.js
call: runWhatsAppGroupActivationAlwaysScenario
args: [{ expr: whatsappScenarioContext }]
call: runWhatsAppScenario
args:
- { expr: whatsappScenarioContext }
- { moduleExport: whatsappQaGroupActivationAlwaysScenario }
qa/scenarios/channels/whatsapp-group-agent-message-action-react.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 90000
retryCount: 1
suiteIsolation: isolated
config: { whatsappScenarioId: whatsapp-group-agent-message-action-react }
flow:
module: ./live-transports/whatsapp/scenario-runtime.js
call: runWhatsAppGroupAgentMessageActionReactScenario
args: [{ expr: whatsappScenarioContext }]
call: runWhatsAppScenario
args:
- { expr: whatsappScenarioContext }
- { moduleExport: whatsappQaGroupAgentMessageActionReactScenario }
qa/scenarios/channels/whatsapp-group-agent-message-action-upload-file.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 90000
retryCount: 1
suiteIsolation: isolated
config: { whatsappScenarioId: whatsapp-group-agent-message-action-upload-file }
flow:
module: ./live-transports/whatsapp/scenario-runtime.js
call: runWhatsAppGroupAgentMessageActionUploadFileScenario
args: [{ expr: whatsappScenarioContext }]
call: runWhatsAppScenario
args:
- { expr: whatsappScenarioContext }
- { moduleExport: whatsappQaGroupAgentMessageActionUploadFileScenario }
qa/scenarios/channels/whatsapp-group-allowlist-block.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 8000
retryCount: 1
suiteIsolation: isolated
config: { whatsappScenarioId: whatsapp-group-allowlist-block }
flow:
module: ./live-transports/whatsapp/scenario-runtime.js
call: runWhatsAppGroupAllowlistBlockScenario
args: [{ expr: whatsappScenarioContext }]
call: runWhatsAppScenario
args:
- { expr: whatsappScenarioContext }
- { moduleExport: whatsappQaGroupAllowlistBlockScenario }
qa/scenarios/channels/whatsapp-group-audio-gating.yaml
+6 -3
View File
@@ -2,6 +2,8 @@ title: WhatsApp group audio mention gating
scenario:
id: whatsapp-group-audio-gating
surface: channels
plugins:
- openai
coverage:
primary:
- whatsapp.group-allowlists
@@ -12,8 +14,9 @@ scenario:
timeoutMs: 120000
retryCount: 1
suiteIsolation: isolated
config: { whatsappScenarioId: whatsapp-group-audio-gating }
flow:
module: ./live-transports/whatsapp/scenario-runtime.js
call: runWhatsAppGroupAudioGatingScenario
args: [{ expr: whatsappScenarioContext }]
call: runWhatsAppScenario
args:
- { expr: whatsappScenarioContext }
- { moduleExport: whatsappQaGroupAudioGatingScenario }
qa/scenarios/channels/whatsapp-group-outbound-audio.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 90000
retryCount: 1
suiteIsolation: isolated
config: { whatsappScenarioId: whatsapp-group-outbound-audio }
flow:
module: ./live-transports/whatsapp/scenario-runtime.js
call: runWhatsAppGroupOutboundAudioScenario
args: [{ expr: whatsappScenarioContext }]
call: runWhatsAppScenario
args:
- { expr: whatsappScenarioContext }
- { moduleExport: whatsappQaGroupOutboundAudioScenario }
qa/scenarios/channels/whatsapp-group-outbound-media.yaml
+4 -3
View File
@@ -12,8 +12,9 @@ scenario:
timeoutMs: 120000
retryCount: 1
suiteIsolation: isolated
config: { whatsappScenarioId: whatsapp-group-outbound-media }
flow:
module: ./live-transports/whatsapp/scenario-runtime.js
call: runWhatsAppGroupOutboundMediaScenario
args: [{ expr: whatsappScenarioContext }]
call: runWhatsAppScenario
args:
- { expr: whatsappScenarioContext }
- { moduleExport: whatsappQaGroupOutboundMediaScenario }
qa/scenarios/channels/whatsapp-group-outbound-poll.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 90000
retryCount: 1
suiteIsolation: isolated
config: { whatsappScenarioId: whatsapp-group-outbound-poll }
flow:
module: ./live-transports/whatsapp/scenario-runtime.js
call: runWhatsAppGroupOutboundPollScenario
args: [{ expr: whatsappScenarioContext }]
call: runWhatsAppScenario
args:
- { expr: whatsappScenarioContext }
- { moduleExport: whatsappQaGroupOutboundPollScenario }
qa/scenarios/channels/whatsapp-group-pending-history-context.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 90000
retryCount: 1
suiteIsolation: isolated
config: { whatsappScenarioId: whatsapp-group-pending-history-context }
flow:
module: ./live-transports/whatsapp/scenario-runtime.js
call: runWhatsAppGroupPendingHistoryContextScenario
args: [{ expr: whatsappScenarioContext }]
call: runWhatsAppScenario
args:
- { expr: whatsappScenarioContext }
- { moduleExport: whatsappQaGroupPendingHistoryContextScenario }
qa/scenarios/channels/whatsapp-group-reply-to-bot-triggers.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 120000
retryCount: 1
suiteIsolation: isolated
config: { whatsappScenarioId: whatsapp-group-reply-to-bot-triggers }
flow:
module: ./live-transports/whatsapp/scenario-runtime.js
call: runWhatsAppGroupReplyToBotTriggersScenario
args: [{ expr: whatsappScenarioContext }]
call: runWhatsAppScenario
args:
- { expr: whatsappScenarioContext }
- { moduleExport: whatsappQaGroupReplyToBotTriggersScenario }
qa/scenarios/channels/whatsapp-group-reply-to-message.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 60000
retryCount: 1
suiteIsolation: isolated
config: { whatsappScenarioId: whatsapp-group-reply-to-message }
flow:
module: ./live-transports/whatsapp/scenario-runtime.js
call: runWhatsAppGroupReplyToMessageScenario
args: [{ expr: whatsappScenarioContext }]
call: runWhatsAppScenario
args:
- { expr: whatsappScenarioContext }
- { moduleExport: whatsappQaGroupReplyToMessageScenario }
qa/scenarios/channels/whatsapp-inbound-image-caption.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 60000
retryCount: 1
suiteIsolation: isolated
config: { whatsappScenarioId: whatsapp-inbound-image-caption }
flow:
module: ./live-transports/whatsapp/scenario-runtime.js
call: runWhatsAppInboundImageCaptionScenario
args: [{ expr: whatsappScenarioContext }]
call: runWhatsAppScenario
args:
- { expr: whatsappScenarioContext }
- { moduleExport: whatsappQaInboundImageCaptionScenario }
qa/scenarios/channels/whatsapp-inbound-reaction-no-trigger.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 90000
retryCount: 1
suiteIsolation: isolated
config: { whatsappScenarioId: whatsapp-inbound-reaction-no-trigger }
flow:
module: ./live-transports/whatsapp/scenario-runtime.js
call: runWhatsAppInboundReactionNoTriggerScenario
args: [{ expr: whatsappScenarioContext }]
call: runWhatsAppScenario
args:
- { expr: whatsappScenarioContext }
- { moduleExport: whatsappQaInboundReactionNoTriggerScenario }
qa/scenarios/channels/whatsapp-inbound-structured-messages.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 240000
retryCount: 1
suiteIsolation: isolated
config: { whatsappScenarioId: whatsapp-inbound-structured-messages }
flow:
module: ./live-transports/whatsapp/scenario-runtime.js
call: runWhatsAppInboundStructuredMessagesScenario
args: [{ expr: whatsappScenarioContext }]
call: runWhatsAppScenario
args:
- { expr: whatsappScenarioContext }
- { moduleExport: whatsappQaInboundStructuredMessagesScenario }
qa/scenarios/channels/whatsapp-mention-gating.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 60000
retryCount: 1
suiteIsolation: isolated
config: { whatsappScenarioId: whatsapp-mention-gating }
flow:
module: ./live-transports/whatsapp/scenario-runtime.js
call: runWhatsAppMentionGatingScenario
args: [{ expr: whatsappScenarioContext }]
call: runWhatsAppScenario
args:
- { expr: whatsappScenarioContext }
- { moduleExport: whatsappQaMentionGatingScenario }
qa/scenarios/channels/whatsapp-message-actions.yaml
+4 -3
View File
@@ -12,8 +12,9 @@ scenario:
timeoutMs: 120000
retryCount: 1
suiteIsolation: isolated
config: { whatsappScenarioId: whatsapp-message-actions }
flow:
module: ./live-transports/whatsapp/scenario-runtime.js
call: runWhatsAppMessageActionsScenario
args: [{ expr: whatsappScenarioContext }]
call: runWhatsAppScenario
args:
- { expr: whatsappScenarioContext }
- { moduleExport: whatsappQaMessageActionsScenario }
qa/scenarios/channels/whatsapp-outbound-document-preserves-filename.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 90000
retryCount: 1
suiteIsolation: isolated
config: { whatsappScenarioId: whatsapp-outbound-document-preserves-filename }
flow:
module: ./live-transports/whatsapp/scenario-runtime.js
call: runWhatsAppOutboundDocumentPreservesFilenameScenario
args: [{ expr: whatsappScenarioContext }]
call: runWhatsAppScenario
args:
- { expr: whatsappScenarioContext }
- { moduleExport: whatsappQaOutboundDocumentPreservesFilenameScenario }
qa/scenarios/channels/whatsapp-outbound-media-matrix.yaml
+4 -3
View File
@@ -13,8 +13,9 @@ scenario:
timeoutMs: 120000
retryCount: 1
suiteIsolation: isolated
config: { whatsappScenarioId: whatsapp-outbound-media-matrix }
flow:
module: ./live-transports/whatsapp/scenario-runtime.js
call: runWhatsAppOutboundMediaMatrixScenario
args: [{ expr: whatsappScenarioContext }]
call: runWhatsAppScenario
args:
- { expr: whatsappScenarioContext }
- { moduleExport: whatsappQaOutboundMediaMatrixScenario }
qa/scenarios/channels/whatsapp-outbound-poll.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 90000
retryCount: 1
suiteIsolation: isolated
config: { whatsappScenarioId: whatsapp-outbound-poll }
flow:
module: ./live-transports/whatsapp/scenario-runtime.js
call: runWhatsAppOutboundPollScenario
args: [{ expr: whatsappScenarioContext }]
call: runWhatsAppScenario
args:
- { expr: whatsappScenarioContext }
- { moduleExport: whatsappQaOutboundPollScenario }
qa/scenarios/channels/whatsapp-outbound-send-serialization.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 90000
retryCount: 1
suiteIsolation: isolated
config: { whatsappScenarioId: whatsapp-outbound-send-serialization }
flow:
module: ./live-transports/whatsapp/scenario-runtime.js
call: runWhatsAppOutboundSendSerializationScenario
args: [{ expr: whatsappScenarioContext }]
call: runWhatsAppScenario
args:
- { expr: whatsappScenarioContext }
- { moduleExport: whatsappQaOutboundSendSerializationScenario }
qa/scenarios/channels/whatsapp-reply-context-isolation.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 120000
retryCount: 1
suiteIsolation: isolated
config: { whatsappScenarioId: whatsapp-reply-context-isolation }
flow:
module: ./live-transports/whatsapp/scenario-runtime.js
call: runWhatsAppReplyContextIsolationScenario
args: [{ expr: whatsappScenarioContext }]
call: runWhatsAppScenario
args:
- { expr: whatsappScenarioContext }
- { moduleExport: whatsappQaReplyContextIsolationScenario }
qa/scenarios/channels/whatsapp-reply-delivery-shape.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 120000
retryCount: 1
suiteIsolation: isolated
config: { whatsappScenarioId: whatsapp-reply-delivery-shape }
flow:
module: ./live-transports/whatsapp/scenario-runtime.js
call: runWhatsAppReplyDeliveryShapeScenario
args: [{ expr: whatsappScenarioContext }]
call: runWhatsAppScenario
args:
- { expr: whatsappScenarioContext }
- { moduleExport: whatsappQaReplyDeliveryShapeScenario }
qa/scenarios/channels/whatsapp-reply-to-message.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 60000
retryCount: 1
suiteIsolation: isolated
config: { whatsappScenarioId: whatsapp-reply-to-message }
flow:
module: ./live-transports/whatsapp/scenario-runtime.js
call: runWhatsAppReplyToMessageScenario
args: [{ expr: whatsappScenarioContext }]
call: runWhatsAppScenario
args:
- { expr: whatsappScenarioContext }
- { moduleExport: whatsappQaReplyToMessageScenario }
qa/scenarios/channels/whatsapp-reply-to-mode-batched.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 90000
retryCount: 1
suiteIsolation: isolated
config: { whatsappScenarioId: whatsapp-reply-to-mode-batched }
flow:
module: ./live-transports/whatsapp/scenario-runtime.js
call: runWhatsAppReplyToModeBatchedScenario
args: [{ expr: whatsappScenarioContext }]
call: runWhatsAppScenario
args:
- { expr: whatsappScenarioContext }
- { moduleExport: whatsappQaReplyToModeBatchedScenario }
qa/scenarios/channels/whatsapp-status-reaction-lifecycle.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 90000
retryCount: 1
suiteIsolation: isolated
config: { whatsappScenarioId: whatsapp-status-reaction-lifecycle }
flow:
module: ./live-transports/whatsapp/scenario-runtime.js
call: runWhatsAppStatusReactionLifecycleScenario
args: [{ expr: whatsappScenarioContext }]
call: runWhatsAppScenario
args:
- { expr: whatsappScenarioContext }
- { moduleExport: whatsappQaStatusReactionLifecycleScenario }
qa/scenarios/channels/whatsapp-status-reactions.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 60000
retryCount: 1
suiteIsolation: isolated
config: { whatsappScenarioId: whatsapp-status-reactions }
flow:
module: ./live-transports/whatsapp/scenario-runtime.js
call: runWhatsAppStatusReactionsScenario
args: [{ expr: whatsappScenarioContext }]
call: runWhatsAppScenario
args:
- { expr: whatsappScenarioContext }
- { moduleExport: whatsappQaStatusReactionsScenario }
qa/scenarios/channels/whatsapp-stream-final-message-accounting.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 90000
retryCount: 1
suiteIsolation: isolated
config: { whatsappScenarioId: whatsapp-stream-final-message-accounting }
flow:
module: ./live-transports/whatsapp/scenario-runtime.js
call: runWhatsAppStreamFinalMessageAccountingScenario
args: [{ expr: whatsappScenarioContext }]
call: runWhatsAppScenario
args:
- { expr: whatsappScenarioContext }
- { moduleExport: whatsappQaStreamFinalMessageAccountingScenario }
qa/scenarios/channels/whatsapp-top-level-reply-shape.yaml
+4 -3
View File
@@ -11,8 +11,9 @@ scenario:
timeoutMs: 60000
retryCount: 1
suiteIsolation: isolated
config: { whatsappScenarioId: whatsapp-top-level-reply-shape }
flow:
module: ./live-transports/whatsapp/scenario-runtime.js
call: runWhatsAppTopLevelReplyShapeScenario
args: [{ expr: whatsappScenarioContext }]
call: runWhatsAppScenario
args:
- { expr: whatsappScenarioContext }
- { moduleExport: whatsappQaTopLevelReplyShapeScenario }
src/plugin-sdk/qa-runner-runtime.ts
+2
View File
@@ -42,6 +42,8 @@ type QaRunnerMessageRecorder = {
type QaRunnerTransportFlowPreparationInput = {
config: Record<string, unknown>;
scenarioId: string;
scenarioTitle: string;
gateway: {
baseUrl: string;
tempRoot: string;