From 3d38c9a6335b701b3b91ba902b4942367c7f4133 Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Sun, 14 Jun 2026 20:51:38 -0700 Subject: [PATCH] test(qa): embed profile scorecard evidence (#93109) * test(qa): embed profile scorecard evidence * test(qa): fix profile runner return lint * test(qa): satisfy suite command lint return --- docs/concepts/qa-e2e-automation.md | 10 +- docs/help/testing.md | 2 + extensions/qa-lab/src/cli.runtime.test.ts | 97 +++++++++- extensions/qa-lab/src/cli.runtime.ts | 28 ++- extensions/qa-lab/src/evidence-summary.ts | 62 +++++++ extensions/qa-lab/src/scorecard-evidence.ts | 190 ++++++++++++++++++++ extensions/qa-lab/src/scorecard-taxonomy.ts | 10 ++ 7 files changed, 391 insertions(+), 8 deletions(-) create mode 100644 extensions/qa-lab/src/scorecard-evidence.ts diff --git a/docs/concepts/qa-e2e-automation.md b/docs/concepts/qa-e2e-automation.md index 5c2841c322d1..00c2dbda918a 100644 --- a/docs/concepts/qa-e2e-automation.md +++ b/docs/concepts/qa-e2e-automation.md @@ -53,7 +53,11 @@ script aliases; both forms are supported. Profile-backed `qa run` reads membership from `taxonomy.yaml`, then dispatches the resolved scenarios through `qa suite`. `--surface` and -`--category` filter the selected profile instead of defining separate lanes: +`--category` filter the selected profile instead of defining separate lanes. +The resulting `qa-evidence.json` includes a profile scorecard summary with +selected-category counts and missing coverage IDs; the individual evidence +entries remain the source of truth for the tests, coverage roles, artifacts, +and results: ```bash pnpm openclaw qa run \ @@ -937,7 +941,9 @@ Every `qa suite` run writes top-level `qa-evidence.json`, `qa-suite-summary.json`, and `qa-suite-report.md` artifacts for the selected scenario set. Scenarios that declare `execution.kind: vitest` or `execution.kind: playwright` run the matching test path and also write -per-scenario logs. +per-scenario logs. When `qa suite` is reached through +`qa run --qa-profile`, the same `qa-evidence.json` also includes the profile +scorecard summary for the selected taxonomy categories. Treat it as a discovery aid, not a gate replacement; the selected scenario still needs the right provider mode, live transport, Multipass, Testbox, or release lane for the behavior under test. For character and style checks, run the same scenario across multiple live model diff --git a/docs/help/testing.md b/docs/help/testing.md index d577741c1417..7ea1f83f23ab 100644 --- a/docs/help/testing.md +++ b/docs/help/testing.md @@ -148,6 +148,8 @@ inside every shard. - Writes top-level `qa-evidence.json`, `qa-suite-summary.json`, and `qa-suite-report.md` artifacts for the selected scenario set, including mixed flow, Vitest, and Playwright scenario selections. + - When dispatched by `pnpm openclaw qa run --qa-profile `, embeds the + selected taxonomy profile scorecard in the same `qa-evidence.json`. - Runs multiple selected scenarios in parallel by default with isolated gateway workers. `qa-channel` defaults to concurrency 4 (bounded by the selected scenario count). Use `--concurrency ` to tune the worker diff --git a/extensions/qa-lab/src/cli.runtime.test.ts b/extensions/qa-lab/src/cli.runtime.test.ts index 79037c96e404..aaa633d5ff62 100644 --- a/extensions/qa-lab/src/cli.runtime.test.ts +++ b/extensions/qa-lab/src/cli.runtime.test.ts @@ -119,6 +119,15 @@ function expectWriteContains(mock: unknown, fragment: string): void { ).toBe(true); } +function makeQaEvidence(entries: unknown[] = []) { + return { + kind: "openclaw.qa.evidence-summary", + schemaVersion: 2, + generatedAt: "2026-06-14T00:00:00.000Z", + entries, + }; +} + function flowSuiteRuntimeResult(params: { evidencePath?: string; reportPath: string; @@ -178,7 +187,7 @@ describe("qa cli runtime", () => { telegramArtifactsDir = await fs.mkdtemp(path.join(os.tmpdir(), "qa-telegram-runtime-")); telegramSummaryPath = path.join(telegramArtifactsDir, QA_EVIDENCE_FILENAME); await fs.writeFile(suiteReportPath, "# QA Suite Report\n", "utf8"); - await fs.writeFile(suiteEvidencePath, JSON.stringify({ entries: [] }), "utf8"); + await fs.writeFile(suiteEvidencePath, JSON.stringify(makeQaEvidence()), "utf8"); await fs.writeFile( suiteSummaryPath, JSON.stringify({ @@ -301,7 +310,7 @@ describe("qa cli runtime", () => { it("runs selected Playwright scenarios through the suite command", async () => { const evidencePath = path.join(suiteArtifactsDir, "qa-evidence.json"); - await fs.writeFile(evidencePath, JSON.stringify({ entries: [] }), "utf8"); + await fs.writeFile(evidencePath, JSON.stringify(makeQaEvidence()), "utf8"); runQaSuite.mockResolvedValueOnce( unifiedSuiteRuntimeResult({ outputDir: suiteArtifactsDir, @@ -349,6 +358,63 @@ describe("qa cli runtime", () => { try { runQaSuite.mockImplementationOnce(async () => { expect(process.env.OPENCLAW_QA_PROFILE).toBe("smoke-ci"); + await fs.writeFile( + suiteEvidencePath, + JSON.stringify( + makeQaEvidence([ + { + test: { + kind: "qa-scenario", + id: "dm-chat-baseline", + title: "DM baseline conversation", + source: { + path: "qa/scenarios/channels/dm-chat-baseline.yaml", + }, + }, + mapping: { + profile: "smoke-ci", + coverage: [ + { + id: "channels.dm", + role: "primary", + surfaceIds: ["dm"], + categoryIds: ["agent-runtime-and-provider-execution.agent-turn-execution"], + }, + ], + }, + execution: { + runner: "host", + environment: { + ref: null, + os: process.platform, + nodeVersion: process.version, + }, + provider: { + id: "openai", + live: false, + model: { + name: "gpt-5.5", + ref: "mock-openai/gpt-5.5", + }, + fixture: "mock-openai", + }, + channel: { + id: "qa-channel", + live: false, + }, + packageSource: { + kind: "source-checkout", + }, + artifacts: [], + }, + result: { + status: "pass", + }, + }, + ]), + ), + "utf8", + ); return flowSuiteRuntimeResult({ reportPath: suiteReportPath, summaryPath: suiteSummaryPath, @@ -379,7 +445,34 @@ describe("qa cli runtime", () => { expect(suiteArgs.scenarioIds).toEqual(expect.arrayContaining(["dm-chat-baseline"])); expect(suiteArgs.scenarioIds).not.toContain("thinking-slash-model-remap"); expect(process.env.OPENCLAW_QA_PROFILE).toBe("release"); + const evidence = JSON.parse(await fs.readFile(suiteEvidencePath, "utf8")) as { + scorecard?: { + profile?: unknown; + run?: { evidenceEntryCount?: unknown }; + features?: { fulfilled?: unknown }; + categoryReports?: Array<{ + id?: unknown; + features?: { fulfilled?: unknown }; + missingCoverageIds?: unknown; + }>; + }; + }; + expect(evidence.scorecard).toMatchObject({ + profile: "smoke-ci", + run: { + evidenceEntryCount: 1, + }, + }); + expect(evidence.scorecard?.features?.fulfilled).toBe(1); + expect(evidence.scorecard?.categoryReports?.[0]).toMatchObject({ + id: "agent-runtime-and-provider-execution.agent-turn-execution", + features: { + fulfilled: 1, + }, + }); + expect(JSON.stringify(evidence.scorecard)).not.toContain("dm-chat-baseline"); expectWriteContains(stdoutWrite, "QA run profile: smoke-ci; categories: 1; scenarios:"); + expectWriteContains(stdoutWrite, `QA profile scorecard: ${suiteEvidencePath}`); } finally { if (previousProfile === undefined) { delete process.env.OPENCLAW_QA_PROFILE; diff --git a/extensions/qa-lab/src/cli.runtime.ts b/extensions/qa-lab/src/cli.runtime.ts index 0902b88075e2..684f172d6186 100644 --- a/extensions/qa-lab/src/cli.runtime.ts +++ b/extensions/qa-lab/src/cli.runtime.ts @@ -68,6 +68,7 @@ import { type QaRuntimeParityTier, } from "./scenario-catalog.js"; import { resolveQaScenarioPackScenarioIds } from "./scenario-packs.js"; +import { attachQaProfileScorecardEvidenceToFile } from "./scorecard-evidence.js"; import { readQaScorecardTaxonomyReport, type QaScorecardCategoryMappingReport, @@ -663,8 +664,9 @@ export async function runQaProfileCommand(opts: QaProfileCommandOptions) { process.stdout.write( `QA run profile: ${profile}; categories: ${categories.length}; scenarios: ${scenarios.length}\n`, ); + let evidencePath: string | undefined; await withTemporaryQaProfileEnv(profile, async () => { - await runQaSuiteCommand({ + const suiteResult = await runQaSuiteCommand({ repoRoot, outputDir: opts.outputDir, transportId: opts.transportId, @@ -676,7 +678,23 @@ export async function runQaProfileCommand(opts: QaProfileCommandOptions) { concurrency: opts.concurrency, allowFailures: opts.allowFailures, }); + evidencePath = + suiteResult && "evidencePath" in suiteResult ? suiteResult.evidencePath : undefined; }); + if (!evidencePath) { + throw new Error("qa run --qa-profile did not produce qa-evidence.json."); + } + await attachQaProfileScorecardEvidenceToFile({ + evidencePath, + taxonomyReport: scorecardReport, + profile, + filters: { + surface: opts.surface, + category: opts.category, + }, + categories, + }); + process.stdout.write(`QA profile scorecard: ${evidencePath}\n`); } function normalizeQaRunProfile(value: string, profileIds: readonly string[]) { @@ -814,7 +832,7 @@ export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) { process.exitCode = 1; } } - return; + return result; } if (opts.preflight === true) { await runQaParityPreflight({ @@ -825,7 +843,7 @@ export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) { alternateModel, allowFailures, }); - return; + return undefined; } const thinkingDefault = parseQaThinkingLevel("--thinking", opts.thinking); const runtimeResult = await runQaSuiteWithInfraRetry(() => @@ -856,7 +874,7 @@ export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) { if (!allowFailures && result.scenarios.some((scenario) => scenario.status !== "pass")) { process.exitCode = 1; } - return; + return result; } case "flow": { const result = runtimeResult.result; @@ -870,8 +888,10 @@ export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) { if (!allowFailures && blockingScenarioCount > 0) { process.exitCode = 1; } + return result; } } + return undefined; } export async function runQaParityReportCommand(opts: { diff --git a/extensions/qa-lab/src/evidence-summary.ts b/extensions/qa-lab/src/evidence-summary.ts index 3e3605dbd5cd..d1d74831d2ad 100644 --- a/extensions/qa-lab/src/evidence-summary.ts +++ b/extensions/qa-lab/src/evidence-summary.ts @@ -110,6 +110,56 @@ const qaEvidenceMappingSchema = z }) .strict(); +const qaEvidenceScorecardCountSchema = z + .object({ + total: z.number().int().nonnegative(), + fulfilled: z.number().int().nonnegative(), + partial: z.number().int().nonnegative().optional(), + missing: z.number().int().nonnegative(), + fulfillmentPercent: z.number().finite().nonnegative(), + }) + .strict(); + +const qaEvidenceScorecardCategorySchema = z + .object({ + id: nonEmptyStringSchema, + surfaceId: nonEmptyStringSchema, + name: nonEmptyStringSchema, + status: z.enum(["fulfilled", "partial", "missing"]), + features: qaEvidenceScorecardCountSchema.extend({ + secondaryOnly: z.number().int().nonnegative(), + }), + missingCoverageIds: z.array(nonEmptyStringSchema), + }) + .strict(); + +const qaEvidenceScorecardSchema = z + .object({ + kind: z.literal("openclaw.qa.scorecard"), + profile: qaEvidenceProfileIdSchema, + taxonomy: z + .object({ + path: nullableStringSchema, + title: nullableStringSchema, + }) + .strict(), + filters: z + .object({ + surface: nullableStringSchema, + category: nullableStringSchema, + }) + .strict(), + run: z + .object({ + evidenceEntryCount: z.number().int().nonnegative(), + }) + .strict(), + categories: qaEvidenceScorecardCountSchema, + features: qaEvidenceScorecardCountSchema, + categoryReports: z.array(qaEvidenceScorecardCategorySchema), + }) + .strict(); + const qaEvidenceArtifactSchema = z .object({ kind: nonEmptyStringSchema, @@ -152,6 +202,7 @@ export const qaEvidenceSummarySchema = z schemaVersion: z.literal(QA_EVIDENCE_SUMMARY_SCHEMA_VERSION), generatedAt: nonEmptyStringSchema, entries: z.array(qaEvidenceSummaryEntrySchema), + scorecard: qaEvidenceScorecardSchema.optional(), }) .strict(); @@ -159,6 +210,7 @@ export type QaEvidenceProfile = z.infer; export type QaEvidenceStatus = z.infer; export type QaEvidenceTiming = z.infer; export type QaEvidencePackageSource = z.infer; +export type QaEvidenceScorecardJson = z.infer; export type QaEvidenceSummaryEntry = z.infer; export type QaEvidenceSummaryJson = z.infer; @@ -476,6 +528,16 @@ export function validateQaEvidenceSummaryJson(summary: unknown): QaEvidenceSumma return qaEvidenceSummarySchema.parse(summary); } +export function attachQaEvidenceScorecard(params: { + summary: QaEvidenceSummaryJson; + scorecard: QaEvidenceScorecardJson; +}): QaEvidenceSummaryJson { + return validateQaEvidenceSummaryJson({ + ...params.summary, + scorecard: params.scorecard, + }); +} + export function buildQaSuiteEvidenceSummary( params: QaEvidenceBuildBase & { channelId: string; diff --git a/extensions/qa-lab/src/scorecard-evidence.ts b/extensions/qa-lab/src/scorecard-evidence.ts new file mode 100644 index 000000000000..75e4ef90e56c --- /dev/null +++ b/extensions/qa-lab/src/scorecard-evidence.ts @@ -0,0 +1,190 @@ +// Qa Lab plugin module embeds profile scorecard context into QA evidence. +import fs from "node:fs/promises"; +import { + attachQaEvidenceScorecard, + validateQaEvidenceSummaryJson, + type QaEvidenceScorecardJson, + type QaEvidenceSummaryEntry, + type QaEvidenceSummaryJson, +} from "./evidence-summary.js"; +import type { + QaScorecardCategoryMappingReport, + QaScorecardTaxonomyReport, +} from "./scorecard-taxonomy.js"; +import { readQaScorecardFeatureCoverageByCategory } from "./scorecard-taxonomy.js"; + +type QaProfileScorecardFilters = { + surface?: string; + category?: string; +}; + +type EvidenceCoverageRole = QaEvidenceSummaryEntry["mapping"]["coverage"][number]["role"]; + +function uniqueSortedStrings(values: Iterable) { + return [ + ...new Set([...values].map((value) => value?.trim()).filter(Boolean) as string[]), + ].toSorted((left, right) => left.localeCompare(right)); +} + +function percent(part: number, total: number) { + return total === 0 ? 0 : Number(((part / total) * 100).toFixed(1)); +} + +function nullableFilter(value: string | undefined) { + const normalized = value?.trim(); + return normalized ? normalized : null; +} + +function coverageIdsForRole( + entries: readonly QaEvidenceSummaryEntry[], + role: EvidenceCoverageRole, +) { + return new Set( + entries.flatMap((entry) => + entry.mapping.coverage + .filter((coverage) => coverage.role === role) + .map((coverage) => coverage.id), + ), + ); +} + +function statusForCategory(params: { featureCount: number; fulfilledFeatureCount: number }) { + if (params.fulfilledFeatureCount === 0) { + return "missing" as const; + } + if (params.fulfilledFeatureCount === params.featureCount) { + return "fulfilled" as const; + } + return "partial" as const; +} + +function categoryFeatureCoverageIds(params: { + category: QaScorecardCategoryMappingReport; + featureCoverageByCategoryId?: ReadonlyMap; +}) { + const features = params.featureCoverageByCategoryId?.get(params.category.id); + return features && features.length > 0 + ? features + : params.category.coverageIds.map((coverageId) => [coverageId]); +} + +export function buildQaProfileScorecardEvidence(params: { + evidence: QaEvidenceSummaryJson; + taxonomyReport: QaScorecardTaxonomyReport; + profile: string; + filters: QaProfileScorecardFilters; + categories: readonly QaScorecardCategoryMappingReport[]; + featureCoverageByCategoryId?: ReadonlyMap; +}): QaEvidenceScorecardJson { + const primaryCoverageIds = coverageIdsForRole(params.evidence.entries, "primary"); + const secondaryCoverageIds = coverageIdsForRole(params.evidence.entries, "secondary"); + const categoryReports = params.categories.map((category) => { + const featureCoverageIds = categoryFeatureCoverageIds({ + category, + featureCoverageByCategoryId: params.featureCoverageByCategoryId, + }); + const fulfilledFeatureCount = featureCoverageIds.filter((coverageIds) => + coverageIds.some((coverageId) => primaryCoverageIds.has(coverageId)), + ).length; + const secondaryOnlyFeatureCount = featureCoverageIds.filter( + (coverageIds) => + !coverageIds.some((coverageId) => primaryCoverageIds.has(coverageId)) && + coverageIds.some((coverageId) => secondaryCoverageIds.has(coverageId)), + ).length; + const missingCoverageIds = uniqueSortedStrings( + featureCoverageIds + .filter( + (coverageIds) => !coverageIds.some((coverageId) => primaryCoverageIds.has(coverageId)), + ) + .flat(), + ); + const missingFeatureCount = featureCoverageIds.length - fulfilledFeatureCount; + return { + id: category.id, + surfaceId: category.taxonomySurfaceId, + name: category.taxonomyCategoryName, + status: statusForCategory({ + featureCount: featureCoverageIds.length, + fulfilledFeatureCount, + }), + features: { + total: featureCoverageIds.length, + fulfilled: fulfilledFeatureCount, + secondaryOnly: secondaryOnlyFeatureCount, + missing: missingFeatureCount, + fulfillmentPercent: percent(fulfilledFeatureCount, featureCoverageIds.length), + }, + missingCoverageIds, + }; + }); + const featureCount = categoryReports.reduce((sum, category) => sum + category.features.total, 0); + const fulfilledFeatureCount = categoryReports.reduce( + (sum, category) => sum + category.features.fulfilled, + 0, + ); + const missingFeatureCount = categoryReports.reduce( + (sum, category) => sum + category.features.missing, + 0, + ); + const fulfilledCategoryCount = categoryReports.filter( + (category) => category.status === "fulfilled", + ).length; + const partialCategoryCount = categoryReports.filter( + (category) => category.status === "partial", + ).length; + const missingCategoryCount = categoryReports.filter( + (category) => category.status === "missing", + ).length; + return { + kind: "openclaw.qa.scorecard", + profile: params.profile, + taxonomy: { + path: params.taxonomyReport.taxonomyPath, + title: params.taxonomyReport.title, + }, + filters: { + surface: nullableFilter(params.filters.surface), + category: nullableFilter(params.filters.category), + }, + run: { + evidenceEntryCount: params.evidence.entries.length, + }, + categories: { + total: categoryReports.length, + fulfilled: fulfilledCategoryCount, + partial: partialCategoryCount, + missing: missingCategoryCount, + fulfillmentPercent: percent(fulfilledCategoryCount, categoryReports.length), + }, + features: { + total: featureCount, + fulfilled: fulfilledFeatureCount, + missing: missingFeatureCount, + fulfillmentPercent: percent(fulfilledFeatureCount, featureCount), + }, + categoryReports, + }; +} + +export async function attachQaProfileScorecardEvidenceToFile(params: { + evidencePath: string; + taxonomyReport: QaScorecardTaxonomyReport; + profile: string; + filters: QaProfileScorecardFilters; + categories: readonly QaScorecardCategoryMappingReport[]; +}) { + const evidence = validateQaEvidenceSummaryJson( + JSON.parse(await fs.readFile(params.evidencePath, "utf8")), + ); + const scorecard = buildQaProfileScorecardEvidence({ + evidence, + taxonomyReport: params.taxonomyReport, + profile: params.profile, + filters: params.filters, + categories: params.categories, + featureCoverageByCategoryId: readQaScorecardFeatureCoverageByCategory(), + }); + const nextEvidence = attachQaEvidenceScorecard({ summary: evidence, scorecard }); + await fs.writeFile(params.evidencePath, `${JSON.stringify(nextEvidence, null, 2)}\n`, "utf8"); + return scorecard; +} diff --git a/extensions/qa-lab/src/scorecard-taxonomy.ts b/extensions/qa-lab/src/scorecard-taxonomy.ts index aea8111d2d03..660e2594bdb7 100644 --- a/extensions/qa-lab/src/scorecard-taxonomy.ts +++ b/extensions/qa-lab/src/scorecard-taxonomy.ts @@ -322,6 +322,16 @@ function buildMaturityRefs(taxonomy: QaMaturityTaxonomy | null) { return { categories, coverageIds }; } +export function readQaScorecardFeatureCoverageByCategory(repoRoot?: string) { + const maturityRefs = buildMaturityRefs(readQaMaturityTaxonomy(repoRoot)); + return new Map( + [...maturityRefs.categories.entries()].map(([categoryId, category]) => [ + categoryId, + category.features.map((feature) => feature.coverageIds), + ]), + ); +} + function pushMissingPrimaryIssues(params: { issues: QaScorecardValidationIssue[]; category: MaturityCategoryRef;