diff --git a/test/e2e/qa-lab/media/hosted-media-provider-live.test.ts b/test/e2e/qa-lab/media/hosted-media-provider-live.test.ts index 0b1ee5e30796..91e2cf592799 100644 --- a/test/e2e/qa-lab/media/hosted-media-provider-live.test.ts +++ b/test/e2e/qa-lab/media/hosted-media-provider-live.test.ts @@ -80,7 +80,7 @@ describe("hosted media provider live QA producer", () => { expect(classifyHostedMediaFailureStatus("provider response was malformed")).toBe("fail"); }); - it("leaves video provider coverage mapping to the scenario catalog", () => { + it("binds video provider coverage from the scenario catalog", () => { const artifactBase = path.join(os.tmpdir(), "openclaw-hosted-media-live-test"); const options = parseHostedMediaOptions(["--suite", "video", "--artifact-base", artifactBase]); const evidence = buildHostedMediaEvidence({ @@ -91,7 +91,11 @@ describe("hosted media provider live QA producer", () => { }, }); - expect(evidence.entries[0]?.coverage).toEqual([]); + expect(evidence.entries[0]?.coverage).toEqual([ + { id: "hosted-providers.video-generation-providers", role: "primary" }, + { id: "media.reference-image-video-and-audio-inputs", role: "primary" }, + { id: "media.video-generation-tool-invocation", role: "secondary" }, + ]); }); }); diff --git a/test/e2e/qa-lab/runtime/browser-plugin-profiles-packaged.ts b/test/e2e/qa-lab/runtime/browser-plugin-profiles-packaged.ts index 96fe713dadf5..93987b6c8332 100644 --- a/test/e2e/qa-lab/runtime/browser-plugin-profiles-packaged.ts +++ b/test/e2e/qa-lab/runtime/browser-plugin-profiles-packaged.ts @@ -11,7 +11,6 @@ const SCENARIO_ID = "browser-plugin-profiles-packaged"; const SOURCE_PATH = "test/e2e/qa-lab/runtime/browser-plugin-profiles-packaged.ts"; const SCRIPT_PATH = "scripts/e2e/browser-plugin-profiles-docker.sh"; const SUCCESS_MARKER = "BROWSER_PLUGIN_PROFILES_PACKAGED_OK"; -const PRIMARY_COVERAGE_IDS = ["tools.browser-plugin-service", "tools.profiles"] as const; type ProducerOptions = { artifactBase: string; repoRoot: string }; type DockerOutcome = { @@ -89,7 +88,6 @@ async function runProducer(options: ProducerOptions): Promise { + vi.unstubAllEnvs(); await Promise.all(tempDirs.splice(0).map((dir) => fs.rm(dir, { force: true, recursive: true }))); }); @@ -17,13 +20,54 @@ describe("diagnostic events boundary runtime", () => { it("composes async dispatch, trusted subscriptions, and model-call trace propagation", async () => { const artifactBase = await fs.mkdtemp(path.join(os.tmpdir(), "diagnostic-events-boundary-")); tempDirs.push(artifactBase); + const repoRoot = process.cwd(); + const checkoutSha = execFileSync("git", ["rev-parse", "--verify", "HEAD"], { + cwd: repoRoot, + encoding: "utf8", + }).trim(); + vi.stubEnv("OPENCLAW_QA_REF", checkoutSha); + vi.stubEnv("OPENCLAW_QA_PACKAGE_SOURCE_KIND", "source-checkout"); + vi.stubEnv("OPENCLAW_QA_PACKAGE_SOURCE_SHA", checkoutSha); const { evidence, summary } = await runDiagnosticEventsBoundaryRuntime({ artifactBase, - repoRoot: process.cwd(), + repoRoot, }); - expect(evidence.entries[0]?.result.status).toBe("pass"); + const diskEvidence = validateQaEvidenceSummaryJson( + JSON.parse(await fs.readFile(path.join(artifactBase, "qa-evidence.json"), "utf8")), + ); + expect(diskEvidence).toEqual(evidence); + expect(evidence.schemaVersion).toBe(2); + expect(evidence.entries).toHaveLength(1); + expect(evidence.entries[0]).toMatchObject({ + coverage: [ + { id: "observability.async-dispatch", role: "primary" }, + { id: "observability.diagnostic-event-types", role: "primary" }, + { id: "observability.model-call-diagnostic-events", role: "primary" }, + { id: "observability.plugin-sdk-diagnostic-runtime-exports", role: "primary" }, + { id: "observability.trusted-diagnostic-event-subscription", role: "primary" }, + { id: "observability.trusted-trace-context", role: "primary" }, + { id: "observability.w3c-trace-context-creation", role: "primary" }, + ], + execution: { + artifacts: [ + { kind: "log", path: "diagnostic-events-boundary.log", source: "script" }, + { + kind: "summary", + path: "diagnostic-events-boundary-summary.json", + source: "script", + }, + ], + environment: { ref: checkoutSha }, + packageSource: { kind: "source-checkout", sha: checkoutSha }, + }, + result: { status: "pass" }, + test: { + id: "diagnostic-events-boundary", + source: { path: "test/e2e/qa-lab/runtime/diagnostic-events-boundary-runtime.ts" }, + }, + }); expect(summary).toMatchObject({ deliveredBeforeDrain: 0, eventTypes: ["model.call.started", "model.call.completed"], @@ -43,7 +87,6 @@ describe("diagnostic events boundary runtime", () => { "utf8", ); expect(summaryText).not.toContain("diagnostic-boundary-private-input"); - await expect(fs.stat(path.join(artifactBase, "qa-evidence.json"))).resolves.toBeDefined(); }); it("rejects missing output-dir values", () => { diff --git a/test/e2e/qa-lab/runtime/script-evidence.test.ts b/test/e2e/qa-lab/runtime/script-evidence.test.ts index 32b5dc09bf40..c7fc4dfafb81 100644 --- a/test/e2e/qa-lab/runtime/script-evidence.test.ts +++ b/test/e2e/qa-lab/runtime/script-evidence.test.ts @@ -19,6 +19,7 @@ async function makeWriter(params: { maxDetailsBytes?: number; maxLogBytes?: numb repoRoot, writer: createQaScriptEvidenceWriter({ artifactBase: path.join(repoRoot, ".artifacts", "qa-e2e", "script"), + coverageBinding: "none", logFileName: "producer.log", maxDetailsBytes: params.maxDetailsBytes, maxLogBytes: params.maxLogBytes ?? 64, @@ -58,6 +59,7 @@ describe("QA script evidence writer", () => { }); expect(evidence.entries[0]).toMatchObject({ + coverage: [], execution: { artifacts: [ { kind: "log", path: "producer.log", source: "script" }, @@ -79,6 +81,23 @@ describe("QA script evidence writer", () => { }); } + it("rejects uncataloged targets unless coverage binding is disabled", () => { + expect(() => + createQaScriptEvidenceWriter({ + artifactBase: path.join(os.tmpdir(), "openclaw-script-evidence-unknown"), + logFileName: "producer.log", + primaryModel: "mock-openai/gpt-5.6-luna", + providerMode: "mock-openai", + repoRoot: process.cwd(), + target: { + id: "script-evidence-test", + sourcePath: "test/e2e/qa-lab/runtime/script-evidence.test.ts", + title: "Script evidence test", + }, + }), + ).toThrow("unknown qa scenario: script-evidence-test"); + }); + it("keeps only the bounded log tail", async () => { const { artifactBase, writer } = await makeWriter({ maxLogBytes: 24 }); writer.appendLog(`discard-me-${"x".repeat(64)}`); diff --git a/test/e2e/qa-lab/runtime/script-evidence.ts b/test/e2e/qa-lab/runtime/script-evidence.ts index 4f0f192a4176..b27df0432934 100644 --- a/test/e2e/qa-lab/runtime/script-evidence.ts +++ b/test/e2e/qa-lab/runtime/script-evidence.ts @@ -4,6 +4,7 @@ import path from "node:path"; import { buildScriptEvidenceSummary, QA_EVIDENCE_FILENAME, + readQaScenarioById, type QaEvidencePackageSource, type QaEvidenceStatus, type QaEvidenceSummaryJson, @@ -44,6 +45,7 @@ type QaScriptEvidenceResult = { type QaScriptEvidenceWriterOptions = { artifactBase: string; + coverageBinding?: "catalog" | "none"; env?: NodeJS.ProcessEnv; evidenceMode?: "full" | "slim"; logFileName: string; @@ -146,6 +148,18 @@ export function createQaScriptBlockedStatusTracker(blockedPatterns: readonly Reg } export function createQaScriptEvidenceWriter(options: QaScriptEvidenceWriterOptions) { + const coverage = + options.coverageBinding === "none" ? undefined : readQaScenarioById(options.target.id).coverage; + // The catalog owns semantic coverage; producers own execution facts only. + const evidenceTarget = { + codeRefs: options.target.codeRefs, + docsRefs: options.target.docsRefs, + id: options.target.id, + primaryCoverageIds: coverage?.primary ?? [], + secondaryCoverageIds: coverage?.secondary ?? [], + sourcePath: options.target.sourcePath, + title: options.target.title, + }; const maxLogBytes = resolveByteLimit(options.maxLogBytes, DEFAULT_CHILD_OUTPUT_TAIL_BYTES); const logFile = resolveArtifactPath(options.artifactBase, options.logFileName); const maxDetailsBytes = resolveByteLimit( @@ -201,7 +215,7 @@ export function createQaScriptEvidenceWriter(options: QaScriptEvidenceWriterOpti providerMode: options.providerMode, repoRoot: options.repoRoot, runner: "script", - targets: [options.target], + targets: [evidenceTarget], results: [ { id: options.target.id, diff --git a/test/e2e/qa-lab/tui/tui-pty-evidence-producer.test.ts b/test/e2e/qa-lab/tui/tui-pty-evidence-producer.test.ts index fce2c1ebaab0..46c47de57faa 100644 --- a/test/e2e/qa-lab/tui/tui-pty-evidence-producer.test.ts +++ b/test/e2e/qa-lab/tui/tui-pty-evidence-producer.test.ts @@ -54,7 +54,7 @@ function makeScenario( config, }; return { - id: "tui-pty-producer-test", + id: "tui-pty-evidence-producer-contract", title: "TUI PTY producer test", surface: "tui", objective: "Prove the TUI PTY evidence producer contract.", diff --git a/test/e2e/qa-lab/tui/tui-pty-evidence-producer.ts b/test/e2e/qa-lab/tui/tui-pty-evidence-producer.ts index 2b056f66b33f..973315831844 100644 --- a/test/e2e/qa-lab/tui/tui-pty-evidence-producer.ts +++ b/test/e2e/qa-lab/tui/tui-pty-evidence-producer.ts @@ -446,8 +446,6 @@ export async function runTuiPtyEvidenceProducer( codeRefs: scenario.codeRefs, docsRefs: scenario.docsRefs, id: scenario.id, - primaryCoverageIds: scenario.coverage?.primary ?? [], - secondaryCoverageIds: scenario.coverage?.secondary ?? [], sourcePath: SOURCE_PATH, title: scenario.title, };