From f5c345b3fe467d9605b4df8399e4e46ba34a4dec Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 7 Jun 2026 00:57:51 +0200 Subject: [PATCH] fix(test): require native live shard proof --- scripts/test-live-shard.mjs | 72 +++++++++++++++++++++++++++- test/scripts/test-live-shard.test.ts | 42 ++++++++++++++++ 2 files changed, 113 insertions(+), 1 deletion(-) diff --git a/scripts/test-live-shard.mjs b/scripts/test-live-shard.mjs index 52c0c2b2afcf..151bea43d8bc 100644 --- a/scripts/test-live-shard.mjs +++ b/scripts/test-live-shard.mjs @@ -331,6 +331,67 @@ export function buildLiveShardPnpmArgs(files, passthroughArgs) { return ["test:live", "--", ...files, ...passthroughArgs]; } +/** + * Builds the Vitest JSON report path used to prove that a live shard ran tests. + */ +export function buildLiveShardReportPath(shard, env = process.env) { + const reportDir = env.OPENCLAW_LIVE_SHARD_REPORT_DIR || ".artifacts/live-shards"; + return path.join(reportDir, `${shard}.vitest.json`); +} + +/** + * Adds reporters needed for both operator logs and machine-readable evidence. + */ +export function addLiveShardReportArgs(passthroughArgs, reportPath) { + return [ + ...passthroughArgs, + "--reporter=default", + "--reporter=json", + `--outputFile.json=${reportPath}`, + ]; +} + +function readNonNegativeInt(value, label) { + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) { + throw new Error(`Vitest report ${label} must be a non-negative integer.`); + } + return value; +} + +/** + * Validates a Vitest JSON payload for live-shard proof. + */ +export function validateLiveShardReportPayload(payload) { + if (!payload || typeof payload !== "object") { + return { ok: false, reason: "Vitest report is not an object." }; + } + let passed; + try { + passed = readNonNegativeInt(payload.numPassedTests, "numPassedTests"); + readNonNegativeInt(payload.numTotalTests, "numTotalTests"); + } catch (error) { + return { ok: false, reason: error instanceof Error ? error.message : String(error) }; + } + if (passed < 1) { + return { ok: false, reason: "Vitest report has no passing live tests." }; + } + return { ok: true }; +} + +/** + * Reads and validates the live-shard Vitest JSON report. + */ +export function validateLiveShardReport(reportPath) { + let payload; + try { + payload = JSON.parse(fs.readFileSync(reportPath, "utf8")); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return { ok: false, reason: `Unable to read Vitest report ${reportPath}: ${message}` }; + } + return validateLiveShardReportPayload(payload); +} + /** * Builds spawn options for the live-shard Vitest child. */ @@ -386,8 +447,10 @@ if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.me } console.log(`[test:live:shard] ${shard}: ${files.length} file(s)`); + const reportPath = buildLiveShardReportPath(shard, process.env); + fs.mkdirSync(path.dirname(reportPath), { recursive: true }); const child = spawnPnpmRunner({ - pnpmArgs: buildLiveShardPnpmArgs(files, passthroughArgs), + pnpmArgs: buildLiveShardPnpmArgs(files, addLiveShardReportArgs(passthroughArgs, reportPath)), ...buildLiveShardSpawnParams(process.env), }); let forwardedSignal = null; @@ -407,6 +470,13 @@ if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.me process.kill(process.pid, forwardedSignal); return; } + if ((code ?? 1) === 0) { + const validation = validateLiveShardReport(reportPath); + if (!validation.ok) { + process.stderr.write(`[test:live:shard] ${validation.reason}\n`); + process.exit(1); + } + } process.exit(code ?? 1); }); child.on("error", (error) => { diff --git a/test/scripts/test-live-shard.test.ts b/test/scripts/test-live-shard.test.ts index 2eeb20e46c49..efee3df8b738 100644 --- a/test/scripts/test-live-shard.test.ts +++ b/test/scripts/test-live-shard.test.ts @@ -5,11 +5,14 @@ import { describe, expect, it } from "vitest"; import { LIVE_TEST_SHARDS, RELEASE_LIVE_TEST_SHARDS, + addLiveShardReportArgs, buildLiveShardPnpmArgs, + buildLiveShardReportPath, buildLiveShardSpawnParams, collectAllLiveTestFiles, parseLiveShardArgs, selectLiveShardFiles, + validateLiveShardReportPayload, } from "../../scripts/test-live-shard.mjs"; import { expectNoReaddirSyncDuring } from "../../src/test-utils/fs-scan-assertions.js"; @@ -167,6 +170,45 @@ describe("scripts/test-live-shard", () => { ]); }); + it("adds JSON report evidence without dropping operator output", () => { + const reportPath = buildLiveShardReportPath("native-live-src-agents", { + OPENCLAW_LIVE_SHARD_REPORT_DIR: ".artifacts/live-proof", + }); + + expect(reportPath).toBe(".artifacts/live-proof/native-live-src-agents.vitest.json"); + expect(addLiveShardReportArgs(["-t", "smoke"], reportPath)).toEqual([ + "-t", + "smoke", + "--reporter=default", + "--reporter=json", + "--outputFile.json=.artifacts/live-proof/native-live-src-agents.vitest.json", + ]); + expect( + buildLiveShardPnpmArgs( + ["src/agents/xai.live.test.ts"], + addLiveShardReportArgs([], reportPath), + ), + ).toContain("--reporter=json"); + }); + + it("fails live shard reports with no passing tests", () => { + expect(validateLiveShardReportPayload({ numPassedTests: 1, numTotalTests: 3 })).toEqual({ + ok: true, + }); + expect(validateLiveShardReportPayload({ numPassedTests: 0, numTotalTests: 3 })).toEqual({ + ok: false, + reason: "Vitest report has no passing live tests.", + }); + expect(validateLiveShardReportPayload({ numPassedTests: 0, numTotalTests: 0 })).toEqual({ + ok: false, + reason: "Vitest report has no passing live tests.", + }); + expect(validateLiveShardReportPayload({ numPassedTests: 0 })).toEqual({ + ok: false, + reason: "Vitest report numTotalTests must be a non-negative integer.", + }); + }); + it("spawns live shard children in a cleanup-friendly process group", () => { expect(buildLiveShardSpawnParams({ PATH: "/usr/bin" }, "darwin")).toEqual({ detached: true,