From 03a7e5b000c444ef6d8fa439e945f7ca22bb93d3 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 25 Aug 2026 06:37:39 -0700 Subject: [PATCH] fix(qa): generate runnable scenario coverage commands (#129340) --- extensions/qa-lab/src/coverage-report.test.ts | 61 +++++++++ extensions/qa-lab/src/coverage-report.ts | 129 +++++++++--------- 2 files changed, 123 insertions(+), 67 deletions(-) diff --git a/extensions/qa-lab/src/coverage-report.test.ts b/extensions/qa-lab/src/coverage-report.test.ts index dda23a88d0c7..81e42799587b 100644 --- a/extensions/qa-lab/src/coverage-report.test.ts +++ b/extensions/qa-lab/src/coverage-report.test.ts @@ -497,6 +497,28 @@ describe("qa coverage report", () => { ]); }); + it.each([ + ["buzz", "channel-canary"], + ["telegram", "channel-canary"], + ["telegram", "channel-workspace-relative-media"], + ["matrix", "channel-workspace-relative-media"], + ] as const)("finds the %s transport declared by %s", (channel, scenarioId) => { + const match = findQaScenarioMatches(readQaScenarioPack().scenarios, channel).find( + (candidate) => candidate.id === scenarioId, + ); + + expect(match).toMatchObject({ id: scenarioId, channel }); + }); + + it.each([ + ["active-memory-preprompt-recall", "mock-openai"], + ["discord-transcripts-voice-authorization", "live-frontier"], + ] as const)("reports the canonical provider mode for %s", (scenarioId, providerMode) => { + expect(findQaScenarioMatches(readQaScenarioPack().scenarios, scenarioId)).toContainEqual( + expect.objectContaining({ id: scenarioId, requiredProviderMode: providerMode }), + ); + }); + it.each([ ["Gateway loopback and LAN access", "docker-gateway-network"], ["qa-lab", "docker-gateway-network"], @@ -569,6 +591,45 @@ describe("qa coverage report", () => { ); }); + it.each([ + [ + "agent-startup-instruction-first-action", + "--provider-mode mock-openai --scenario agent-startup-instruction-first-action", + ], + [ + "channel-workspace-relative-media", + "--channel-driver live --channel matrix --scenario channel-workspace-relative-media", + ], + ["channel-canary", "--scenario channel-canary"], + ] as const)("renders a runnable suite command for %s", (scenarioId, expectedArgs) => { + const matches = findQaScenarioMatches(readQaScenarioPack().scenarios, scenarioId); + const report = renderQaScenarioMatchesMarkdownReport({ query: scenarioId, matches }); + + expect(report).toContain(`- Suite command: \`pnpm openclaw qa suite ${expectedArgs}\``); + }); + + it("groups commands by compatible provider mode while preserving the live default", () => { + const scenarios = readQaScenarioPack().scenarios; + const scenarioIds = [ + "agent-startup-instruction-first-action", + "instruction-profile-artifact-followthrough-live", + "instruction-followthrough-repo-contract", + ]; + const matches = scenarioIds.flatMap((scenarioId) => + findQaScenarioMatches(scenarios, scenarioId), + ); + const report = renderQaScenarioMatchesMarkdownReport({ query: "provider lanes", matches }); + + expect(report).toContain("- Suite commands:"); + expect(report).toContain( + "--provider-mode mock-openai --scenario agent-startup-instruction-first-action", + ); + expect(report).toContain( + "--scenario instruction-profile-artifact-followthrough-live --scenario instruction-followthrough-repo-contract", + ); + expect(report).not.toContain("--provider-mode live-frontier"); + }); + it("splits flow commands across channel lanes", () => { const scenarios = readQaScenarioPack().scenarios; const matches = [ diff --git a/extensions/qa-lab/src/coverage-report.ts b/extensions/qa-lab/src/coverage-report.ts index 9e90250da71a..e0a99b879b8c 100644 --- a/extensions/qa-lab/src/coverage-report.ts +++ b/extensions/qa-lab/src/coverage-report.ts @@ -4,7 +4,11 @@ import { normalizeStringEntriesLower, } from "openclaw/plugin-sdk/string-coerce-runtime"; import { isRepoRootRelativeRef } from "./cli-paths.js"; -import type { QaSeedScenarioWithSource } from "./scenario-catalog.js"; +import { DEFAULT_QA_LIVE_PROVIDER_MODE } from "./providers/index.js"; +import { + resolveQaScenarioRequiredProviderMode, + type QaSeedScenarioWithSource, +} from "./scenario-catalog.js"; import { readQaScorecardTaxonomyReport, type QaScorecardTaxonomyReport, @@ -77,69 +81,54 @@ function assertUniqueQaScenarioIds( } } -function scenarioTheme(sourcePath: string) { - const parts = sourcePath.split("/"); - return parts[2] ?? "unknown"; -} - -function scenarioSurfaces(scenario: QaSeedScenarioWithSource) { - return scenario.surfaces && scenario.surfaces.length > 0 ? scenario.surfaces : [scenario.surface]; -} - -function scenarioRisk(scenario: QaSeedScenarioWithSource) { - return scenario.risk ?? scenario.riskLevel ?? "unassigned"; -} - function summarizeScenario(scenario: QaSeedScenarioWithSource): QaCoverageScenarioSummary { return { id: scenario.id, title: scenario.title, sourcePath: scenario.sourcePath, - theme: scenarioTheme(scenario.sourcePath), - surfaces: scenarioSurfaces(scenario), - risk: scenarioRisk(scenario), + theme: scenario.sourcePath.split("/")[2] ?? "unknown", + surfaces: scenario.surfaces?.length ? scenario.surfaces : [scenario.surface], + risk: scenario.risk ?? scenario.riskLevel ?? "unassigned", }; } -function normalizeSearchText(value: string) { - return value.toLowerCase(); -} - -function tokenizeScenarioSearchQuery(query: string) { - return normalizeStringEntriesLower(query.split(/\s+/u)); -} - function scenarioSearchText(scenario: QaSeedScenarioWithSource) { const config = scenario.execution.config ?? {}; - return normalizeSearchText( - [ - scenario.id, - scenario.title, - scenario.sourcePath, - scenario.surface, - ...(scenario.surfaces ?? []), - scenario.category ?? "", - scenario.runtimePairLane ?? "", - scenario.risk ?? "", - scenario.riskLevel ?? "", - scenario.objective, - ...scenario.successCriteria, - ...(scenario.capabilities ?? []), - ...(scenario.plugins ?? []), - ...(scenario.docsRefs ?? []), - ...(scenario.codeRefs ?? []), - ...(scenario.coverage?.primary ?? []), - ...(scenario.coverage?.secondary ?? []), - ...Object.entries(config).flatMap(([key, value]) => [ - key, - typeof value === "string" ? value : "", - ]), - ].join("\n"), - ); + return [ + scenario.id, + scenario.title, + scenario.sourcePath, + scenario.surface, + ...(scenario.surfaces ?? []), + scenario.category ?? "", + scenario.runtimePairLane ?? "", + scenario.risk ?? "", + scenario.riskLevel ?? "", + scenario.objective, + ...scenario.successCriteria, + ...(scenario.capabilities ?? []), + ...(scenario.plugins ?? []), + ...(scenario.execution.channels ?? []), + resolveQaScenarioRequiredProviderMode(scenario) ?? "", + ...(scenario.docsRefs ?? []), + ...(scenario.codeRefs ?? []), + ...(scenario.coverage?.primary ?? []), + ...(scenario.coverage?.secondary ?? []), + ...Object.entries(config).flatMap(([key, value]) => [ + key, + typeof value === "string" ? value : "", + ]), + ] + .join("\n") + .toLowerCase(); } -function summarizeScenarioSearchMatch(scenario: QaSeedScenarioWithSource): QaScenarioSearchMatch { +function summarizeScenarioSearchMatch( + scenario: QaSeedScenarioWithSource, + tokens: readonly string[], +): QaScenarioSearchMatch { const config = scenario.execution.config ?? {}; + const channels = scenario.execution.channels ?? []; return { ...summarizeScenario(scenario), coverageIds: [ @@ -149,10 +138,14 @@ function summarizeScenarioSearchMatch(scenario: QaSeedScenarioWithSource): QaSce docsRefs: [...(scenario.docsRefs ?? [])], codeRefs: [...(scenario.codeRefs ?? [])], executionKind: scenario.execution.kind, - channel: scenario.execution.channel, + channel: + channels.find((channel) => tokens.includes(channel)) ?? + scenario.execution.channel ?? + channels.find((channel) => channel === "qa-channel") ?? + channels[0], ...(scenario.execution.kind !== "flow" ? { executionPath: scenario.execution.path } : {}), runtimePairLane: scenario.runtimePairLane, - requiredProviderMode: stringifyConfigValue(config.requiredProviderMode), + requiredProviderMode: resolveQaScenarioRequiredProviderMode(scenario), requiredProvider: stringifyConfigValue(config.requiredProvider), requiredModel: stringifyConfigValue(config.requiredModel), }; @@ -162,7 +155,7 @@ export function findQaScenarioMatches( scenarios: readonly QaSeedScenarioWithSource[], query: string, ) { - const tokens = tokenizeScenarioSearchQuery(query); + const tokens = normalizeStringEntriesLower(query.split(/\s+/u)); if (tokens.length === 0) { return []; } @@ -178,11 +171,11 @@ export function findQaScenarioMatches( executionPathQuery.includes("/") && isRepoRootRelativeRef(executionPathQuery) && scenario.execution.kind !== "flow" && - normalizeSearchText(scenario.execution.path).includes(executionPathQuery) + scenario.execution.path.toLowerCase().includes(executionPathQuery) ); }); }) - .map(summarizeScenarioSearchMatch) + .map((scenario) => summarizeScenarioSearchMatch(scenario, tokens)) .toSorted((left, right) => left.id.localeCompare(right.id)); } @@ -398,24 +391,26 @@ function formatOptionalScenarioMetadata(match: QaScenarioSearchMatch) { return metadata.length > 0 ? metadata.join("; ") : "none"; } -function uniqueScenarioValues(values: (string | undefined)[]) { - return [...new Set(values.filter((value): value is string => Boolean(value)))]; -} - function formatSuiteCommand(matches: readonly QaScenarioSearchMatch[]) { const scenarioArgs = matches.map((match) => `--scenario ${match.id}`).join(" "); - const channels = uniqueScenarioValues(matches.map((match) => match.channel)); - const [channel] = channels; - const selectedDriver = channels.length === 1 && channel !== "qa-channel" ? "live" : undefined; - const driverArg = selectedDriver ? ` --channel-driver ${selectedDriver}` : ""; - const channelArg = driverArg && channel ? ` --channel ${channel}` : ""; - return `pnpm openclaw qa suite${driverArg}${channelArg} ${scenarioArgs}`; + const { channel, requiredProviderMode } = matches[0]!; + const channelArg = + channel && channel !== "qa-channel" ? ` --channel-driver live --channel ${channel}` : ""; + const providerModeArg = + requiredProviderMode && requiredProviderMode !== DEFAULT_QA_LIVE_PROVIDER_MODE + ? ` --provider-mode ${requiredProviderMode}` + : ""; + return `pnpm openclaw qa suite${channelArg}${providerModeArg} ${scenarioArgs}`; } function scenarioMatchCommandGroups(matches: readonly QaScenarioSearchMatch[]) { const groups = new Map(); for (const match of matches) { - const key = JSON.stringify([match.executionKind, match.channel]); + const key = JSON.stringify([ + match.executionKind, + match.channel, + match.requiredProviderMode ?? DEFAULT_QA_LIVE_PROVIDER_MODE, + ]); const group = groups.get(key) ?? []; group.push(match); groups.set(key, group);