fix(qa): generate runnable scenario coverage commands (#129340)

extensions/qa-lab/src/coverage-report.test.ts

@@ -497,6 +497,28 @@ describe("qa coverage report", () => {
     ]);
   });
 
+  it.each([
+    ["buzz", "channel-canary"],
+    ["telegram", "channel-canary"],
+    ["telegram", "channel-workspace-relative-media"],
+    ["matrix", "channel-workspace-relative-media"],
+  ] as const)("finds the %s transport declared by %s", (channel, scenarioId) => {
+    const match = findQaScenarioMatches(readQaScenarioPack().scenarios, channel).find(
+      (candidate) => candidate.id === scenarioId,
+    );
+
+    expect(match).toMatchObject({ id: scenarioId, channel });
+  });
+
+  it.each([
+    ["active-memory-preprompt-recall", "mock-openai"],
+    ["discord-transcripts-voice-authorization", "live-frontier"],
+  ] as const)("reports the canonical provider mode for %s", (scenarioId, providerMode) => {
+    expect(findQaScenarioMatches(readQaScenarioPack().scenarios, scenarioId)).toContainEqual(
+      expect.objectContaining({ id: scenarioId, requiredProviderMode: providerMode }),
+    );
+  });
+
   it.each([
     ["Gateway loopback and LAN access", "docker-gateway-network"],
     ["qa-lab", "docker-gateway-network"],
@@ -569,6 +591,45 @@ describe("qa coverage report", () => {
     );
   });
 
+  it.each([
+    [
+      "agent-startup-instruction-first-action",
+      "--provider-mode mock-openai --scenario agent-startup-instruction-first-action",
+    ],
+    [
+      "channel-workspace-relative-media",
+      "--channel-driver live --channel matrix --scenario channel-workspace-relative-media",
+    ],
+    ["channel-canary", "--scenario channel-canary"],
+  ] as const)("renders a runnable suite command for %s", (scenarioId, expectedArgs) => {
+    const matches = findQaScenarioMatches(readQaScenarioPack().scenarios, scenarioId);
+    const report = renderQaScenarioMatchesMarkdownReport({ query: scenarioId, matches });
+
+    expect(report).toContain(`- Suite command: \`pnpm openclaw qa suite ${expectedArgs}\``);
+  });
+
+  it("groups commands by compatible provider mode while preserving the live default", () => {
+    const scenarios = readQaScenarioPack().scenarios;
+    const scenarioIds = [
+      "agent-startup-instruction-first-action",
+      "instruction-profile-artifact-followthrough-live",
+      "instruction-followthrough-repo-contract",
+    ];
+    const matches = scenarioIds.flatMap((scenarioId) =>
+      findQaScenarioMatches(scenarios, scenarioId),
+    );
+    const report = renderQaScenarioMatchesMarkdownReport({ query: "provider lanes", matches });
+
+    expect(report).toContain("- Suite commands:");
+    expect(report).toContain(
+      "--provider-mode mock-openai --scenario agent-startup-instruction-first-action",
+    );
+    expect(report).toContain(
+      "--scenario instruction-profile-artifact-followthrough-live --scenario instruction-followthrough-repo-contract",
+    );
+    expect(report).not.toContain("--provider-mode live-frontier");
+  });
+
   it("splits flow commands across channel lanes", () => {
     const scenarios = readQaScenarioPack().scenarios;
     const matches = [

extensions/qa-lab/src/coverage-report.ts

@@ -4,7 +4,11 @@ import {
   normalizeStringEntriesLower,
 } from "openclaw/plugin-sdk/string-coerce-runtime";
 import { isRepoRootRelativeRef } from "./cli-paths.js";
-import type { QaSeedScenarioWithSource } from "./scenario-catalog.js";
+import { DEFAULT_QA_LIVE_PROVIDER_MODE } from "./providers/index.js";
+import {
+  resolveQaScenarioRequiredProviderMode,
+  type QaSeedScenarioWithSource,
+} from "./scenario-catalog.js";
 import {
   readQaScorecardTaxonomyReport,
   type QaScorecardTaxonomyReport,
@@ -77,69 +81,54 @@ function assertUniqueQaScenarioIds(
   }
 }
 
-function scenarioTheme(sourcePath: string) {
-  const parts = sourcePath.split("/");
-  return parts[2] ?? "unknown";
-}
-
-function scenarioSurfaces(scenario: QaSeedScenarioWithSource) {
-  return scenario.surfaces && scenario.surfaces.length > 0 ? scenario.surfaces : [scenario.surface];
-}
-
-function scenarioRisk(scenario: QaSeedScenarioWithSource) {
-  return scenario.risk ?? scenario.riskLevel ?? "unassigned";
-}
-
 function summarizeScenario(scenario: QaSeedScenarioWithSource): QaCoverageScenarioSummary {
   return {
     id: scenario.id,
     title: scenario.title,
     sourcePath: scenario.sourcePath,
-    theme: scenarioTheme(scenario.sourcePath),
-    surfaces: scenarioSurfaces(scenario),
-    risk: scenarioRisk(scenario),
+    theme: scenario.sourcePath.split("/")[2] ?? "unknown",
+    surfaces: scenario.surfaces?.length ? scenario.surfaces : [scenario.surface],
+    risk: scenario.risk ?? scenario.riskLevel ?? "unassigned",
   };
 }
 
-function normalizeSearchText(value: string) {
-  return value.toLowerCase();
-}
-
-function tokenizeScenarioSearchQuery(query: string) {
-  return normalizeStringEntriesLower(query.split(/\s+/u));
-}
-
 function scenarioSearchText(scenario: QaSeedScenarioWithSource) {
   const config = scenario.execution.config ?? {};
-  return normalizeSearchText(
-    [
-      scenario.id,
-      scenario.title,
-      scenario.sourcePath,
-      scenario.surface,
-      ...(scenario.surfaces ?? []),
-      scenario.category ?? "",
-      scenario.runtimePairLane ?? "",
-      scenario.risk ?? "",
-      scenario.riskLevel ?? "",
-      scenario.objective,
-      ...scenario.successCriteria,
-      ...(scenario.capabilities ?? []),
-      ...(scenario.plugins ?? []),
-      ...(scenario.docsRefs ?? []),
-      ...(scenario.codeRefs ?? []),
-      ...(scenario.coverage?.primary ?? []),
-      ...(scenario.coverage?.secondary ?? []),
-      ...Object.entries(config).flatMap(([key, value]) => [
-        key,
-        typeof value === "string" ? value : "",
-      ]),
-    ].join("\n"),
-  );
+  return [
+    scenario.id,
+    scenario.title,
+    scenario.sourcePath,
+    scenario.surface,
+    ...(scenario.surfaces ?? []),
+    scenario.category ?? "",
+    scenario.runtimePairLane ?? "",
+    scenario.risk ?? "",
+    scenario.riskLevel ?? "",
+    scenario.objective,
+    ...scenario.successCriteria,
+    ...(scenario.capabilities ?? []),
+    ...(scenario.plugins ?? []),
+    ...(scenario.execution.channels ?? []),
+    resolveQaScenarioRequiredProviderMode(scenario) ?? "",
+    ...(scenario.docsRefs ?? []),
+    ...(scenario.codeRefs ?? []),
+    ...(scenario.coverage?.primary ?? []),
+    ...(scenario.coverage?.secondary ?? []),
+    ...Object.entries(config).flatMap(([key, value]) => [
+      key,
+      typeof value === "string" ? value : "",
+    ]),
+  ]
+    .join("\n")
+    .toLowerCase();
 }
 
-function summarizeScenarioSearchMatch(scenario: QaSeedScenarioWithSource): QaScenarioSearchMatch {
+function summarizeScenarioSearchMatch(
+  scenario: QaSeedScenarioWithSource,
+  tokens: readonly string[],
+): QaScenarioSearchMatch {
   const config = scenario.execution.config ?? {};
+  const channels = scenario.execution.channels ?? [];
   return {
     ...summarizeScenario(scenario),
     coverageIds: [
@@ -149,10 +138,14 @@ function summarizeScenarioSearchMatch(scenario: QaSeedScenarioWithSource): QaSce
     docsRefs: [...(scenario.docsRefs ?? [])],
     codeRefs: [...(scenario.codeRefs ?? [])],
     executionKind: scenario.execution.kind,
-    channel: scenario.execution.channel,
+    channel:
+      channels.find((channel) => tokens.includes(channel)) ??
+      scenario.execution.channel ??
+      channels.find((channel) => channel === "qa-channel") ??
+      channels[0],
     ...(scenario.execution.kind !== "flow" ? { executionPath: scenario.execution.path } : {}),
     runtimePairLane: scenario.runtimePairLane,
-    requiredProviderMode: stringifyConfigValue(config.requiredProviderMode),
+    requiredProviderMode: resolveQaScenarioRequiredProviderMode(scenario),
     requiredProvider: stringifyConfigValue(config.requiredProvider),
     requiredModel: stringifyConfigValue(config.requiredModel),
   };
@@ -162,7 +155,7 @@ export function findQaScenarioMatches(
   scenarios: readonly QaSeedScenarioWithSource[],
   query: string,
 ) {
-  const tokens = tokenizeScenarioSearchQuery(query);
+  const tokens = normalizeStringEntriesLower(query.split(/\s+/u));
   if (tokens.length === 0) {
     return [];
   }
@@ -178,11 +171,11 @@ export function findQaScenarioMatches(
           executionPathQuery.includes("/") &&
           isRepoRootRelativeRef(executionPathQuery) &&
           scenario.execution.kind !== "flow" &&
-          normalizeSearchText(scenario.execution.path).includes(executionPathQuery)
+          scenario.execution.path.toLowerCase().includes(executionPathQuery)
         );
       });
     })
-    .map(summarizeScenarioSearchMatch)
+    .map((scenario) => summarizeScenarioSearchMatch(scenario, tokens))
     .toSorted((left, right) => left.id.localeCompare(right.id));
 }
 
@@ -398,24 +391,26 @@ function formatOptionalScenarioMetadata(match: QaScenarioSearchMatch) {
   return metadata.length > 0 ? metadata.join("; ") : "none";
 }
 
-function uniqueScenarioValues(values: (string | undefined)[]) {
-  return [...new Set(values.filter((value): value is string => Boolean(value)))];
-}
-
 function formatSuiteCommand(matches: readonly QaScenarioSearchMatch[]) {
   const scenarioArgs = matches.map((match) => `--scenario ${match.id}`).join(" ");
-  const channels = uniqueScenarioValues(matches.map((match) => match.channel));
-  const [channel] = channels;
-  const selectedDriver = channels.length === 1 && channel !== "qa-channel" ? "live" : undefined;
-  const driverArg = selectedDriver ? ` --channel-driver ${selectedDriver}` : "";
-  const channelArg = driverArg && channel ? ` --channel ${channel}` : "";
-  return `pnpm openclaw qa suite${driverArg}${channelArg} ${scenarioArgs}`;
+  const { channel, requiredProviderMode } = matches[0]!;
+  const channelArg =
+    channel && channel !== "qa-channel" ? ` --channel-driver live --channel ${channel}` : "";
+  const providerModeArg =
+    requiredProviderMode && requiredProviderMode !== DEFAULT_QA_LIVE_PROVIDER_MODE
+      ? ` --provider-mode ${requiredProviderMode}`
+      : "";
+  return `pnpm openclaw qa suite${channelArg}${providerModeArg} ${scenarioArgs}`;
 }
 
 function scenarioMatchCommandGroups(matches: readonly QaScenarioSearchMatch[]) {
   const groups = new Map<string, QaScenarioSearchMatch[]>();
   for (const match of matches) {
-    const key = JSON.stringify([match.executionKind, match.channel]);
+    const key = JSON.stringify([
+      match.executionKind,
+      match.channel,
+      match.requiredProviderMode ?? DEFAULT_QA_LIVE_PROVIDER_MODE,
+    ]);
     const group = groups.get(key) ?? [];
     group.push(match);
     groups.set(key, group);