fix(qa): preserve script evidence coverage (#120286)

test/e2e/qa-lab/media/hosted-media-provider-live.test.ts

@@ -80,7 +80,7 @@ describe("hosted media provider live QA producer", () => {
     expect(classifyHostedMediaFailureStatus("provider response was malformed")).toBe("fail");
   });
 
-  it("leaves video provider coverage mapping to the scenario catalog", () => {
+  it("binds video provider coverage from the scenario catalog", () => {
     const artifactBase = path.join(os.tmpdir(), "openclaw-hosted-media-live-test");
     const options = parseHostedMediaOptions(["--suite", "video", "--artifact-base", artifactBase]);
     const evidence = buildHostedMediaEvidence({
@@ -91,7 +91,11 @@ describe("hosted media provider live QA producer", () => {
       },
     });
 
-    expect(evidence.entries[0]?.coverage).toEqual([]);
+    expect(evidence.entries[0]?.coverage).toEqual([
+      { id: "hosted-providers.video-generation-providers", role: "primary" },
+      { id: "media.reference-image-video-and-audio-inputs", role: "primary" },
+      { id: "media.video-generation-tool-invocation", role: "secondary" },
+    ]);
   });
 });
 

test/e2e/qa-lab/runtime/browser-plugin-profiles-packaged.ts

@@ -11,7 +11,6 @@ const SCENARIO_ID = "browser-plugin-profiles-packaged";
 const SOURCE_PATH = "test/e2e/qa-lab/runtime/browser-plugin-profiles-packaged.ts";
 const SCRIPT_PATH = "scripts/e2e/browser-plugin-profiles-docker.sh";
 const SUCCESS_MARKER = "BROWSER_PLUGIN_PROFILES_PACKAGED_OK";
-const PRIMARY_COVERAGE_IDS = ["tools.browser-plugin-service", "tools.profiles"] as const;
 
 type ProducerOptions = { artifactBase: string; repoRoot: string };
 type DockerOutcome = {
@@ -89,7 +88,6 @@ async function runProducer(options: ProducerOptions): Promise<QaEvidenceSummaryJ
     codeRefs: [SOURCE_PATH, SCRIPT_PATH, "extensions/browser/src/gateway/browser-request.ts"],
     docsRefs: ["docs/tools/browser.md", "docs/help/testing.md"],
     id: SCENARIO_ID,
-    primaryCoverageIds: PRIMARY_COVERAGE_IDS,
     sourcePath: SOURCE_PATH,
     title: "Packaged browser plugin profiles",
   };

test/e2e/qa-lab/runtime/diagnostic-events-boundary-runtime.test.ts

@@ -1,7 +1,9 @@
+import { execFileSync } from "node:child_process";
 import fs from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
-import { afterEach, describe, expect, it } from "vitest";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { validateQaEvidenceSummaryJson } from "../../../../extensions/qa-lab/api.js";
 import {
   runDiagnosticEventsBoundaryRuntime,
   testing,
@@ -10,6 +12,7 @@ import {
 const tempDirs: string[] = [];
 
 afterEach(async () => {
+  vi.unstubAllEnvs();
   await Promise.all(tempDirs.splice(0).map((dir) => fs.rm(dir, { force: true, recursive: true })));
 });
 
@@ -17,13 +20,54 @@ describe("diagnostic events boundary runtime", () => {
   it("composes async dispatch, trusted subscriptions, and model-call trace propagation", async () => {
     const artifactBase = await fs.mkdtemp(path.join(os.tmpdir(), "diagnostic-events-boundary-"));
     tempDirs.push(artifactBase);
+    const repoRoot = process.cwd();
+    const checkoutSha = execFileSync("git", ["rev-parse", "--verify", "HEAD"], {
+      cwd: repoRoot,
+      encoding: "utf8",
+    }).trim();
+    vi.stubEnv("OPENCLAW_QA_REF", checkoutSha);
+    vi.stubEnv("OPENCLAW_QA_PACKAGE_SOURCE_KIND", "source-checkout");
+    vi.stubEnv("OPENCLAW_QA_PACKAGE_SOURCE_SHA", checkoutSha);
 
     const { evidence, summary } = await runDiagnosticEventsBoundaryRuntime({
       artifactBase,
-      repoRoot: process.cwd(),
+      repoRoot,
     });
 
-    expect(evidence.entries[0]?.result.status).toBe("pass");
+    const diskEvidence = validateQaEvidenceSummaryJson(
+      JSON.parse(await fs.readFile(path.join(artifactBase, "qa-evidence.json"), "utf8")),
+    );
+    expect(diskEvidence).toEqual(evidence);
+    expect(evidence.schemaVersion).toBe(2);
+    expect(evidence.entries).toHaveLength(1);
+    expect(evidence.entries[0]).toMatchObject({
+      coverage: [
+        { id: "observability.async-dispatch", role: "primary" },
+        { id: "observability.diagnostic-event-types", role: "primary" },
+        { id: "observability.model-call-diagnostic-events", role: "primary" },
+        { id: "observability.plugin-sdk-diagnostic-runtime-exports", role: "primary" },
+        { id: "observability.trusted-diagnostic-event-subscription", role: "primary" },
+        { id: "observability.trusted-trace-context", role: "primary" },
+        { id: "observability.w3c-trace-context-creation", role: "primary" },
+      ],
+      execution: {
+        artifacts: [
+          { kind: "log", path: "diagnostic-events-boundary.log", source: "script" },
+          {
+            kind: "summary",
+            path: "diagnostic-events-boundary-summary.json",
+            source: "script",
+          },
+        ],
+        environment: { ref: checkoutSha },
+        packageSource: { kind: "source-checkout", sha: checkoutSha },
+      },
+      result: { status: "pass" },
+      test: {
+        id: "diagnostic-events-boundary",
+        source: { path: "test/e2e/qa-lab/runtime/diagnostic-events-boundary-runtime.ts" },
+      },
+    });
     expect(summary).toMatchObject({
       deliveredBeforeDrain: 0,
       eventTypes: ["model.call.started", "model.call.completed"],
@@ -43,7 +87,6 @@ describe("diagnostic events boundary runtime", () => {
       "utf8",
     );
     expect(summaryText).not.toContain("diagnostic-boundary-private-input");
-    await expect(fs.stat(path.join(artifactBase, "qa-evidence.json"))).resolves.toBeDefined();
   });
 
   it("rejects missing output-dir values", () => {

test/e2e/qa-lab/runtime/script-evidence.test.ts

@@ -19,6 +19,7 @@ async function makeWriter(params: { maxDetailsBytes?: number; maxLogBytes?: numb
     repoRoot,
     writer: createQaScriptEvidenceWriter({
       artifactBase: path.join(repoRoot, ".artifacts", "qa-e2e", "script"),
+      coverageBinding: "none",
       logFileName: "producer.log",
       maxDetailsBytes: params.maxDetailsBytes,
       maxLogBytes: params.maxLogBytes ?? 64,
@@ -58,6 +59,7 @@ describe("QA script evidence writer", () => {
       });
 
       expect(evidence.entries[0]).toMatchObject({
+        coverage: [],
         execution: {
           artifacts: [
             { kind: "log", path: "producer.log", source: "script" },
@@ -79,6 +81,23 @@ describe("QA script evidence writer", () => {
     });
   }
 
+  it("rejects uncataloged targets unless coverage binding is disabled", () => {
+    expect(() =>
+      createQaScriptEvidenceWriter({
+        artifactBase: path.join(os.tmpdir(), "openclaw-script-evidence-unknown"),
+        logFileName: "producer.log",
+        primaryModel: "mock-openai/gpt-5.6-luna",
+        providerMode: "mock-openai",
+        repoRoot: process.cwd(),
+        target: {
+          id: "script-evidence-test",
+          sourcePath: "test/e2e/qa-lab/runtime/script-evidence.test.ts",
+          title: "Script evidence test",
+        },
+      }),
+    ).toThrow("unknown qa scenario: script-evidence-test");
+  });
+
   it("keeps only the bounded log tail", async () => {
     const { artifactBase, writer } = await makeWriter({ maxLogBytes: 24 });
     writer.appendLog(`discard-me-${"x".repeat(64)}`);

test/e2e/qa-lab/runtime/script-evidence.ts

@@ -4,6 +4,7 @@ import path from "node:path";
 import {
   buildScriptEvidenceSummary,
   QA_EVIDENCE_FILENAME,
+  readQaScenarioById,
   type QaEvidencePackageSource,
   type QaEvidenceStatus,
   type QaEvidenceSummaryJson,
@@ -44,6 +45,7 @@ type QaScriptEvidenceResult = {
 
 type QaScriptEvidenceWriterOptions = {
   artifactBase: string;
+  coverageBinding?: "catalog" | "none";
   env?: NodeJS.ProcessEnv;
   evidenceMode?: "full" | "slim";
   logFileName: string;
@@ -146,6 +148,18 @@ export function createQaScriptBlockedStatusTracker(blockedPatterns: readonly Reg
 }
 
 export function createQaScriptEvidenceWriter(options: QaScriptEvidenceWriterOptions) {
+  const coverage =
+    options.coverageBinding === "none" ? undefined : readQaScenarioById(options.target.id).coverage;
+  // The catalog owns semantic coverage; producers own execution facts only.
+  const evidenceTarget = {
+    codeRefs: options.target.codeRefs,
+    docsRefs: options.target.docsRefs,
+    id: options.target.id,
+    primaryCoverageIds: coverage?.primary ?? [],
+    secondaryCoverageIds: coverage?.secondary ?? [],
+    sourcePath: options.target.sourcePath,
+    title: options.target.title,
+  };
   const maxLogBytes = resolveByteLimit(options.maxLogBytes, DEFAULT_CHILD_OUTPUT_TAIL_BYTES);
   const logFile = resolveArtifactPath(options.artifactBase, options.logFileName);
   const maxDetailsBytes = resolveByteLimit(
@@ -201,7 +215,7 @@ export function createQaScriptEvidenceWriter(options: QaScriptEvidenceWriterOpti
       providerMode: options.providerMode,
       repoRoot: options.repoRoot,
       runner: "script",
-      targets: [options.target],
+      targets: [evidenceTarget],
       results: [
         {
           id: options.target.id,

test/e2e/qa-lab/tui/tui-pty-evidence-producer.test.ts

@@ -54,7 +54,7 @@ function makeScenario(
           config,
         };
   return {
-    id: "tui-pty-producer-test",
+    id: "tui-pty-evidence-producer-contract",
     title: "TUI PTY producer test",
     surface: "tui",
     objective: "Prove the TUI PTY evidence producer contract.",

test/e2e/qa-lab/tui/tui-pty-evidence-producer.ts

@@ -446,8 +446,6 @@ export async function runTuiPtyEvidenceProducer(
     codeRefs: scenario.codeRefs,
     docsRefs: scenario.docsRefs,
     id: scenario.id,
-    primaryCoverageIds: scenario.coverage?.primary ?? [],
-    secondaryCoverageIds: scenario.coverage?.secondary ?? [],
     sourcePath: SOURCE_PATH,
     title: scenario.title,
   };