test(qa): embed profile scorecard evidence (#93109)

* test(qa): embed profile scorecard evidence

* test(qa): fix profile runner return lint

* test(qa): satisfy suite command lint return

docs/concepts/qa-e2e-automation.md

@@ -53,7 +53,11 @@ script aliases; both forms are supported.
 
 Profile-backed `qa run` reads membership from `taxonomy.yaml`, then dispatches
 the resolved scenarios through `qa suite`. `--surface` and
-`--category` filter the selected profile instead of defining separate lanes:
+`--category` filter the selected profile instead of defining separate lanes.
+The resulting `qa-evidence.json` includes a profile scorecard summary with
+selected-category counts and missing coverage IDs; the individual evidence
+entries remain the source of truth for the tests, coverage roles, artifacts,
+and results:
 
 ```bash
 pnpm openclaw qa run \
@@ -937,7 +941,9 @@ Every `qa suite` run writes top-level `qa-evidence.json`,
 `qa-suite-summary.json`, and `qa-suite-report.md` artifacts for the selected
 scenario set. Scenarios that declare `execution.kind: vitest` or
 `execution.kind: playwright` run the matching test path and also write
-per-scenario logs.
+per-scenario logs. When `qa suite` is reached through
+`qa run --qa-profile`, the same `qa-evidence.json` also includes the profile
+scorecard summary for the selected taxonomy categories.
 Treat it as a discovery aid, not a gate replacement; the selected scenario still needs the right provider mode, live transport, Multipass, Testbox, or release lane for the behavior under test.
 
 For character and style checks, run the same scenario across multiple live model

docs/help/testing.md

@@ -148,6 +148,8 @@ inside every shard.
   - Writes top-level `qa-evidence.json`, `qa-suite-summary.json`, and
     `qa-suite-report.md` artifacts for the selected scenario set, including
     mixed flow, Vitest, and Playwright scenario selections.
+  - When dispatched by `pnpm openclaw qa run --qa-profile <profile>`, embeds the
+    selected taxonomy profile scorecard in the same `qa-evidence.json`.
   - Runs multiple selected scenarios in parallel by default with isolated
     gateway workers. `qa-channel` defaults to concurrency 4 (bounded by the
     selected scenario count). Use `--concurrency <count>` to tune the worker

extensions/qa-lab/src/cli.runtime.test.ts

@@ -119,6 +119,15 @@ function expectWriteContains(mock: unknown, fragment: string): void {
   ).toBe(true);
 }
 
+function makeQaEvidence(entries: unknown[] = []) {
+  return {
+    kind: "openclaw.qa.evidence-summary",
+    schemaVersion: 2,
+    generatedAt: "2026-06-14T00:00:00.000Z",
+    entries,
+  };
+}
+
 function flowSuiteRuntimeResult(params: {
   evidencePath?: string;
   reportPath: string;
@@ -178,7 +187,7 @@ describe("qa cli runtime", () => {
     telegramArtifactsDir = await fs.mkdtemp(path.join(os.tmpdir(), "qa-telegram-runtime-"));
     telegramSummaryPath = path.join(telegramArtifactsDir, QA_EVIDENCE_FILENAME);
     await fs.writeFile(suiteReportPath, "# QA Suite Report\n", "utf8");
-    await fs.writeFile(suiteEvidencePath, JSON.stringify({ entries: [] }), "utf8");
+    await fs.writeFile(suiteEvidencePath, JSON.stringify(makeQaEvidence()), "utf8");
     await fs.writeFile(
       suiteSummaryPath,
       JSON.stringify({
@@ -301,7 +310,7 @@ describe("qa cli runtime", () => {
 
   it("runs selected Playwright scenarios through the suite command", async () => {
     const evidencePath = path.join(suiteArtifactsDir, "qa-evidence.json");
-    await fs.writeFile(evidencePath, JSON.stringify({ entries: [] }), "utf8");
+    await fs.writeFile(evidencePath, JSON.stringify(makeQaEvidence()), "utf8");
     runQaSuite.mockResolvedValueOnce(
       unifiedSuiteRuntimeResult({
         outputDir: suiteArtifactsDir,
@@ -349,6 +358,63 @@ describe("qa cli runtime", () => {
     try {
       runQaSuite.mockImplementationOnce(async () => {
         expect(process.env.OPENCLAW_QA_PROFILE).toBe("smoke-ci");
+        await fs.writeFile(
+          suiteEvidencePath,
+          JSON.stringify(
+            makeQaEvidence([
+              {
+                test: {
+                  kind: "qa-scenario",
+                  id: "dm-chat-baseline",
+                  title: "DM baseline conversation",
+                  source: {
+                    path: "qa/scenarios/channels/dm-chat-baseline.yaml",
+                  },
+                },
+                mapping: {
+                  profile: "smoke-ci",
+                  coverage: [
+                    {
+                      id: "channels.dm",
+                      role: "primary",
+                      surfaceIds: ["dm"],
+                      categoryIds: ["agent-runtime-and-provider-execution.agent-turn-execution"],
+                    },
+                  ],
+                },
+                execution: {
+                  runner: "host",
+                  environment: {
+                    ref: null,
+                    os: process.platform,
+                    nodeVersion: process.version,
+                  },
+                  provider: {
+                    id: "openai",
+                    live: false,
+                    model: {
+                      name: "gpt-5.5",
+                      ref: "mock-openai/gpt-5.5",
+                    },
+                    fixture: "mock-openai",
+                  },
+                  channel: {
+                    id: "qa-channel",
+                    live: false,
+                  },
+                  packageSource: {
+                    kind: "source-checkout",
+                  },
+                  artifacts: [],
+                },
+                result: {
+                  status: "pass",
+                },
+              },
+            ]),
+          ),
+          "utf8",
+        );
         return flowSuiteRuntimeResult({
           reportPath: suiteReportPath,
           summaryPath: suiteSummaryPath,
@@ -379,7 +445,34 @@ describe("qa cli runtime", () => {
       expect(suiteArgs.scenarioIds).toEqual(expect.arrayContaining(["dm-chat-baseline"]));
       expect(suiteArgs.scenarioIds).not.toContain("thinking-slash-model-remap");
       expect(process.env.OPENCLAW_QA_PROFILE).toBe("release");
+      const evidence = JSON.parse(await fs.readFile(suiteEvidencePath, "utf8")) as {
+        scorecard?: {
+          profile?: unknown;
+          run?: { evidenceEntryCount?: unknown };
+          features?: { fulfilled?: unknown };
+          categoryReports?: Array<{
+            id?: unknown;
+            features?: { fulfilled?: unknown };
+            missingCoverageIds?: unknown;
+          }>;
+        };
+      };
+      expect(evidence.scorecard).toMatchObject({
+        profile: "smoke-ci",
+        run: {
+          evidenceEntryCount: 1,
+        },
+      });
+      expect(evidence.scorecard?.features?.fulfilled).toBe(1);
+      expect(evidence.scorecard?.categoryReports?.[0]).toMatchObject({
+        id: "agent-runtime-and-provider-execution.agent-turn-execution",
+        features: {
+          fulfilled: 1,
+        },
+      });
+      expect(JSON.stringify(evidence.scorecard)).not.toContain("dm-chat-baseline");
       expectWriteContains(stdoutWrite, "QA run profile: smoke-ci; categories: 1; scenarios:");
+      expectWriteContains(stdoutWrite, `QA profile scorecard: ${suiteEvidencePath}`);
     } finally {
       if (previousProfile === undefined) {
         delete process.env.OPENCLAW_QA_PROFILE;

extensions/qa-lab/src/cli.runtime.ts

@@ -68,6 +68,7 @@ import {
   type QaRuntimeParityTier,
 } from "./scenario-catalog.js";
 import { resolveQaScenarioPackScenarioIds } from "./scenario-packs.js";
+import { attachQaProfileScorecardEvidenceToFile } from "./scorecard-evidence.js";
 import {
   readQaScorecardTaxonomyReport,
   type QaScorecardCategoryMappingReport,
@@ -663,8 +664,9 @@ export async function runQaProfileCommand(opts: QaProfileCommandOptions) {
   process.stdout.write(
     `QA run profile: ${profile}; categories: ${categories.length}; scenarios: ${scenarios.length}\n`,
   );
+  let evidencePath: string | undefined;
   await withTemporaryQaProfileEnv(profile, async () => {
-    await runQaSuiteCommand({
+    const suiteResult = await runQaSuiteCommand({
       repoRoot,
       outputDir: opts.outputDir,
       transportId: opts.transportId,
@@ -676,7 +678,23 @@ export async function runQaProfileCommand(opts: QaProfileCommandOptions) {
       concurrency: opts.concurrency,
       allowFailures: opts.allowFailures,
     });
+    evidencePath =
+      suiteResult && "evidencePath" in suiteResult ? suiteResult.evidencePath : undefined;
   });
+  if (!evidencePath) {
+    throw new Error("qa run --qa-profile did not produce qa-evidence.json.");
+  }
+  await attachQaProfileScorecardEvidenceToFile({
+    evidencePath,
+    taxonomyReport: scorecardReport,
+    profile,
+    filters: {
+      surface: opts.surface,
+      category: opts.category,
+    },
+    categories,
+  });
+  process.stdout.write(`QA profile scorecard: ${evidencePath}\n`);
 }
 
 function normalizeQaRunProfile(value: string, profileIds: readonly string[]) {
@@ -814,7 +832,7 @@ export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) {
         process.exitCode = 1;
       }
     }
-    return;
+    return result;
   }
   if (opts.preflight === true) {
     await runQaParityPreflight({
@@ -825,7 +843,7 @@ export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) {
       alternateModel,
       allowFailures,
     });
-    return;
+    return undefined;
   }
   const thinkingDefault = parseQaThinkingLevel("--thinking", opts.thinking);
   const runtimeResult = await runQaSuiteWithInfraRetry(() =>
@@ -856,7 +874,7 @@ export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) {
       if (!allowFailures && result.scenarios.some((scenario) => scenario.status !== "pass")) {
         process.exitCode = 1;
       }
-      return;
+      return result;
     }
     case "flow": {
       const result = runtimeResult.result;
@@ -870,8 +888,10 @@ export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) {
       if (!allowFailures && blockingScenarioCount > 0) {
         process.exitCode = 1;
       }
+      return result;
     }
   }
+  return undefined;
 }
 
 export async function runQaParityReportCommand(opts: {

extensions/qa-lab/src/evidence-summary.ts

@@ -110,6 +110,56 @@ const qaEvidenceMappingSchema = z
   })
   .strict();
 
+const qaEvidenceScorecardCountSchema = z
+  .object({
+    total: z.number().int().nonnegative(),
+    fulfilled: z.number().int().nonnegative(),
+    partial: z.number().int().nonnegative().optional(),
+    missing: z.number().int().nonnegative(),
+    fulfillmentPercent: z.number().finite().nonnegative(),
+  })
+  .strict();
+
+const qaEvidenceScorecardCategorySchema = z
+  .object({
+    id: nonEmptyStringSchema,
+    surfaceId: nonEmptyStringSchema,
+    name: nonEmptyStringSchema,
+    status: z.enum(["fulfilled", "partial", "missing"]),
+    features: qaEvidenceScorecardCountSchema.extend({
+      secondaryOnly: z.number().int().nonnegative(),
+    }),
+    missingCoverageIds: z.array(nonEmptyStringSchema),
+  })
+  .strict();
+
+const qaEvidenceScorecardSchema = z
+  .object({
+    kind: z.literal("openclaw.qa.scorecard"),
+    profile: qaEvidenceProfileIdSchema,
+    taxonomy: z
+      .object({
+        path: nullableStringSchema,
+        title: nullableStringSchema,
+      })
+      .strict(),
+    filters: z
+      .object({
+        surface: nullableStringSchema,
+        category: nullableStringSchema,
+      })
+      .strict(),
+    run: z
+      .object({
+        evidenceEntryCount: z.number().int().nonnegative(),
+      })
+      .strict(),
+    categories: qaEvidenceScorecardCountSchema,
+    features: qaEvidenceScorecardCountSchema,
+    categoryReports: z.array(qaEvidenceScorecardCategorySchema),
+  })
+  .strict();
+
 const qaEvidenceArtifactSchema = z
   .object({
     kind: nonEmptyStringSchema,
@@ -152,6 +202,7 @@ export const qaEvidenceSummarySchema = z
     schemaVersion: z.literal(QA_EVIDENCE_SUMMARY_SCHEMA_VERSION),
     generatedAt: nonEmptyStringSchema,
     entries: z.array(qaEvidenceSummaryEntrySchema),
+    scorecard: qaEvidenceScorecardSchema.optional(),
   })
   .strict();
 
@@ -159,6 +210,7 @@ export type QaEvidenceProfile = z.infer<typeof qaEvidenceProfileIdSchema>;
 export type QaEvidenceStatus = z.infer<typeof qaEvidenceStatusSchema>;
 export type QaEvidenceTiming = z.infer<typeof qaEvidenceTimingSchema>;
 export type QaEvidencePackageSource = z.infer<typeof qaEvidencePackageSourceSchema>;
+export type QaEvidenceScorecardJson = z.infer<typeof qaEvidenceScorecardSchema>;
 export type QaEvidenceSummaryEntry = z.infer<typeof qaEvidenceSummaryEntrySchema>;
 export type QaEvidenceSummaryJson = z.infer<typeof qaEvidenceSummarySchema>;
 
@@ -476,6 +528,16 @@ export function validateQaEvidenceSummaryJson(summary: unknown): QaEvidenceSumma
   return qaEvidenceSummarySchema.parse(summary);
 }
 
+export function attachQaEvidenceScorecard(params: {
+  summary: QaEvidenceSummaryJson;
+  scorecard: QaEvidenceScorecardJson;
+}): QaEvidenceSummaryJson {
+  return validateQaEvidenceSummaryJson({
+    ...params.summary,
+    scorecard: params.scorecard,
+  });
+}
+
 export function buildQaSuiteEvidenceSummary(
   params: QaEvidenceBuildBase & {
     channelId: string;

extensions/qa-lab/src/scorecard-evidence.ts

@@ -0,0 +1,190 @@
+// Qa Lab plugin module embeds profile scorecard context into QA evidence.
+import fs from "node:fs/promises";
+import {
+  attachQaEvidenceScorecard,
+  validateQaEvidenceSummaryJson,
+  type QaEvidenceScorecardJson,
+  type QaEvidenceSummaryEntry,
+  type QaEvidenceSummaryJson,
+} from "./evidence-summary.js";
+import type {
+  QaScorecardCategoryMappingReport,
+  QaScorecardTaxonomyReport,
+} from "./scorecard-taxonomy.js";
+import { readQaScorecardFeatureCoverageByCategory } from "./scorecard-taxonomy.js";
+
+type QaProfileScorecardFilters = {
+  surface?: string;
+  category?: string;
+};
+
+type EvidenceCoverageRole = QaEvidenceSummaryEntry["mapping"]["coverage"][number]["role"];
+
+function uniqueSortedStrings(values: Iterable<string | undefined>) {
+  return [
+    ...new Set([...values].map((value) => value?.trim()).filter(Boolean) as string[]),
+  ].toSorted((left, right) => left.localeCompare(right));
+}
+
+function percent(part: number, total: number) {
+  return total === 0 ? 0 : Number(((part / total) * 100).toFixed(1));
+}
+
+function nullableFilter(value: string | undefined) {
+  const normalized = value?.trim();
+  return normalized ? normalized : null;
+}
+
+function coverageIdsForRole(
+  entries: readonly QaEvidenceSummaryEntry[],
+  role: EvidenceCoverageRole,
+) {
+  return new Set(
+    entries.flatMap((entry) =>
+      entry.mapping.coverage
+        .filter((coverage) => coverage.role === role)
+        .map((coverage) => coverage.id),
+    ),
+  );
+}
+
+function statusForCategory(params: { featureCount: number; fulfilledFeatureCount: number }) {
+  if (params.fulfilledFeatureCount === 0) {
+    return "missing" as const;
+  }
+  if (params.fulfilledFeatureCount === params.featureCount) {
+    return "fulfilled" as const;
+  }
+  return "partial" as const;
+}
+
+function categoryFeatureCoverageIds(params: {
+  category: QaScorecardCategoryMappingReport;
+  featureCoverageByCategoryId?: ReadonlyMap<string, readonly (readonly string[])[]>;
+}) {
+  const features = params.featureCoverageByCategoryId?.get(params.category.id);
+  return features && features.length > 0
+    ? features
+    : params.category.coverageIds.map((coverageId) => [coverageId]);
+}
+
+export function buildQaProfileScorecardEvidence(params: {
+  evidence: QaEvidenceSummaryJson;
+  taxonomyReport: QaScorecardTaxonomyReport;
+  profile: string;
+  filters: QaProfileScorecardFilters;
+  categories: readonly QaScorecardCategoryMappingReport[];
+  featureCoverageByCategoryId?: ReadonlyMap<string, readonly (readonly string[])[]>;
+}): QaEvidenceScorecardJson {
+  const primaryCoverageIds = coverageIdsForRole(params.evidence.entries, "primary");
+  const secondaryCoverageIds = coverageIdsForRole(params.evidence.entries, "secondary");
+  const categoryReports = params.categories.map((category) => {
+    const featureCoverageIds = categoryFeatureCoverageIds({
+      category,
+      featureCoverageByCategoryId: params.featureCoverageByCategoryId,
+    });
+    const fulfilledFeatureCount = featureCoverageIds.filter((coverageIds) =>
+      coverageIds.some((coverageId) => primaryCoverageIds.has(coverageId)),
+    ).length;
+    const secondaryOnlyFeatureCount = featureCoverageIds.filter(
+      (coverageIds) =>
+        !coverageIds.some((coverageId) => primaryCoverageIds.has(coverageId)) &&
+        coverageIds.some((coverageId) => secondaryCoverageIds.has(coverageId)),
+    ).length;
+    const missingCoverageIds = uniqueSortedStrings(
+      featureCoverageIds
+        .filter(
+          (coverageIds) => !coverageIds.some((coverageId) => primaryCoverageIds.has(coverageId)),
+        )
+        .flat(),
+    );
+    const missingFeatureCount = featureCoverageIds.length - fulfilledFeatureCount;
+    return {
+      id: category.id,
+      surfaceId: category.taxonomySurfaceId,
+      name: category.taxonomyCategoryName,
+      status: statusForCategory({
+        featureCount: featureCoverageIds.length,
+        fulfilledFeatureCount,
+      }),
+      features: {
+        total: featureCoverageIds.length,
+        fulfilled: fulfilledFeatureCount,
+        secondaryOnly: secondaryOnlyFeatureCount,
+        missing: missingFeatureCount,
+        fulfillmentPercent: percent(fulfilledFeatureCount, featureCoverageIds.length),
+      },
+      missingCoverageIds,
+    };
+  });
+  const featureCount = categoryReports.reduce((sum, category) => sum + category.features.total, 0);
+  const fulfilledFeatureCount = categoryReports.reduce(
+    (sum, category) => sum + category.features.fulfilled,
+    0,
+  );
+  const missingFeatureCount = categoryReports.reduce(
+    (sum, category) => sum + category.features.missing,
+    0,
+  );
+  const fulfilledCategoryCount = categoryReports.filter(
+    (category) => category.status === "fulfilled",
+  ).length;
+  const partialCategoryCount = categoryReports.filter(
+    (category) => category.status === "partial",
+  ).length;
+  const missingCategoryCount = categoryReports.filter(
+    (category) => category.status === "missing",
+  ).length;
+  return {
+    kind: "openclaw.qa.scorecard",
+    profile: params.profile,
+    taxonomy: {
+      path: params.taxonomyReport.taxonomyPath,
+      title: params.taxonomyReport.title,
+    },
+    filters: {
+      surface: nullableFilter(params.filters.surface),
+      category: nullableFilter(params.filters.category),
+    },
+    run: {
+      evidenceEntryCount: params.evidence.entries.length,
+    },
+    categories: {
+      total: categoryReports.length,
+      fulfilled: fulfilledCategoryCount,
+      partial: partialCategoryCount,
+      missing: missingCategoryCount,
+      fulfillmentPercent: percent(fulfilledCategoryCount, categoryReports.length),
+    },
+    features: {
+      total: featureCount,
+      fulfilled: fulfilledFeatureCount,
+      missing: missingFeatureCount,
+      fulfillmentPercent: percent(fulfilledFeatureCount, featureCount),
+    },
+    categoryReports,
+  };
+}
+
+export async function attachQaProfileScorecardEvidenceToFile(params: {
+  evidencePath: string;
+  taxonomyReport: QaScorecardTaxonomyReport;
+  profile: string;
+  filters: QaProfileScorecardFilters;
+  categories: readonly QaScorecardCategoryMappingReport[];
+}) {
+  const evidence = validateQaEvidenceSummaryJson(
+    JSON.parse(await fs.readFile(params.evidencePath, "utf8")),
+  );
+  const scorecard = buildQaProfileScorecardEvidence({
+    evidence,
+    taxonomyReport: params.taxonomyReport,
+    profile: params.profile,
+    filters: params.filters,
+    categories: params.categories,
+    featureCoverageByCategoryId: readQaScorecardFeatureCoverageByCategory(),
+  });
+  const nextEvidence = attachQaEvidenceScorecard({ summary: evidence, scorecard });
+  await fs.writeFile(params.evidencePath, `${JSON.stringify(nextEvidence, null, 2)}\n`, "utf8");
+  return scorecard;
+}

extensions/qa-lab/src/scorecard-taxonomy.ts

@@ -322,6 +322,16 @@ function buildMaturityRefs(taxonomy: QaMaturityTaxonomy | null) {
   return { categories, coverageIds };
 }
 
+export function readQaScorecardFeatureCoverageByCategory(repoRoot?: string) {
+  const maturityRefs = buildMaturityRefs(readQaMaturityTaxonomy(repoRoot));
+  return new Map(
+    [...maturityRefs.categories.entries()].map(([categoryId, category]) => [
+      categoryId,
+      category.features.map((feature) => feature.coverageIds),
+    ]),
+  );
+}
+
 function pushMissingPrimaryIssues(params: {
   issues: QaScorecardValidationIssue[];
   category: MaturityCategoryRef;