diff --git a/extensions/qa-lab/src/confidence-report.test.ts b/extensions/qa-lab/src/confidence-report.test.ts index 1d28e70e25a0..20f8c4257162 100644 --- a/extensions/qa-lab/src/confidence-report.test.ts +++ b/extensions/qa-lab/src/confidence-report.test.ts @@ -449,39 +449,55 @@ describe("qa confidence report", () => { } }); - it("rejects skipped token reports when a live usage source is required", async () => { - await writeJson("live-token/qa-runtime-token-efficiency-summary.json", { - status: "skipped", - pass: true, - rows: [], - }); + it.each([ + ["skipped", "skipped", [], undefined, false, "token summary has no usage rows"], + ["empty", "estimated", [], undefined, false, "token summary has no usage rows"], + ["missing", "estimated", undefined, undefined, false, "token summary missing rows"], + [ + "executed", + "estimated", + [{ usageSource: "mock-estimate" }], + undefined, + true, + "summary pass=true", + ], + ["live", "skipped", [], "live-usage", false, "token summary has no live-usage rows"], + ] as const)( + "evaluates %s token evidence", + async (_name, status, rows, expectedSource, passed, details) => { + await writeJson("live-token/qa-runtime-token-efficiency-summary.json", { + status, + pass: true, + ...(rows ? { rows } : {}), + }); - const report = await buildQaConfidenceReport({ - manifest: { - version: 1, - profile: "codex-100", - lanes: [ - { - id: "live-token-efficiency", - title: "Live token efficiency", - kind: "token-efficiency-summary", - artifact: "live-token/qa-runtime-token-efficiency-summary.json", - required: true, - expectedTokenUsageSource: "live-usage", - }, - ], - }, - artifactRoot: tempRoot, - strictZeroUnknowns: true, - generatedAt: "2026-05-12T00:00:00.000Z", - }); + const report = await buildQaConfidenceReport({ + manifest: { + version: 1, + profile: "codex-100", + lanes: [ + { + id: "live-token-efficiency", + title: "Live token efficiency", + kind: "token-efficiency-summary", + artifact: "live-token/qa-runtime-token-efficiency-summary.json", + required: true, + ...(expectedSource ? { expectedTokenUsageSource: expectedSource } : {}), + }, + ], + }, + artifactRoot: tempRoot, + strictGlobalPass: true, + }); - expect(report.pass).toBe(false); - expect(report.lanes[0]).toMatchObject({ - status: "unknown", - details: "token summary has no live-usage rows", - }); - }); + expect(report.pass).toBe(passed); + expect(report.globalPass).toBe(passed); + expect(report.lanes[0]).toMatchObject({ + status: passed ? "pass" : "unknown", + details, + }); + }, + ); it("preserves partial zero-unknown mode for classified failing lanes", async () => { await writeJson("classified/qa-suite-summary.json", { diff --git a/extensions/qa-lab/src/confidence-report.ts b/extensions/qa-lab/src/confidence-report.ts index c51b5077617c..baadc0bec0cb 100644 --- a/extensions/qa-lab/src/confidence-report.ts +++ b/extensions/qa-lab/src/confidence-report.ts @@ -549,22 +549,22 @@ function evaluateTokenEfficiencySummary( expectedTokenUsageSource: QaConfidenceManifestLane["expectedTokenUsageSource"], ): QaConfidenceLaneEvaluation { const base = evaluatePassSummary(payload); - if (!base.passed || !expectedTokenUsageSource) { + if (!base.passed || !isRecord(payload)) { return base; } - if (!isRecord(payload) || !Array.isArray(payload.rows)) { + const rows = Array.isArray(payload.rows) ? payload.rows : undefined; + if (!rows || rows.length === 0 || readString(payload.status) === "skipped") { return { passed: false, - details: `token summary missing rows for expected usageSource=${expectedTokenUsageSource}`, + details: !rows + ? `token summary missing rows${expectedTokenUsageSource ? ` for expected usageSource=${expectedTokenUsageSource}` : ""}` + : `token summary has no ${expectedTokenUsageSource ?? "usage"} rows`, }; } - if (readString(payload.status) === "skipped" || payload.rows.length === 0) { - return { - passed: false, - details: `token summary has no ${expectedTokenUsageSource} rows`, - }; + if (!expectedTokenUsageSource) { + return base; } - const mismatched = payload.rows.filter( + const mismatched = rows.filter( (row) => !isRecord(row) || row.usageSource !== expectedTokenUsageSource, ); return {