diff --git a/.github/workflows/openclaw-performance.yml b/.github/workflows/openclaw-performance.yml index 21c5df8febae..f701e5053162 100644 --- a/.github/workflows/openclaw-performance.yml +++ b/.github/workflows/openclaw-performance.yml @@ -519,11 +519,7 @@ jobs: status=${PIPESTATUS[0]} set -e - report_json="$(find "$REPORT_DIR" -maxdepth 1 -type f -name '*.json' -print | sort | tail -n 1)" - if [[ -z "$report_json" ]]; then - echo "Kova did not write a JSON report." >&2 - exit 1 - fi + report_json="$(node "$PERFORMANCE_HELPER_DIR/scripts/lib/kova-report-selector.mjs" --report-dir "$REPORT_DIR")" report_md="${report_json%.json}.md" node "$PERFORMANCE_HELPER_DIR/scripts/lib/kova-workflow-evidence.mjs" \ --plan "$KOVA_PLAN_JSON" \ @@ -586,8 +582,8 @@ jobs: run: | set -euo pipefail missing=0 - if ! find "$REPORT_DIR" -maxdepth 1 -type f -name '*.json' -size +0c -print -quit | grep -q .; then - echo "::error::Kova JSON report is missing for ${LANE_ID}." + if ! node "$PERFORMANCE_HELPER_DIR/scripts/lib/kova-report-selector.mjs" --report-dir "$REPORT_DIR" >/dev/null; then + echo "::error::Exactly one full Kova JSON report is required for ${LANE_ID}." missing=1 fi if [[ ! -s "$BUNDLE_DIR/bundle.json" ]]; then @@ -802,6 +798,7 @@ jobs: env: LANE_ID: ${{ matrix.lane }} PERFORMANCE_PUBLISHER_HELPER: ${{ github.workspace }}/.artifacts/performance-publisher/scripts/lib/kova-report-publish-files.mjs + PERFORMANCE_REPORT_SELECTOR: ${{ github.workspace }}/.artifacts/performance-publisher/scripts/lib/kova-report-selector.mjs PUBLISHED_REPORT_MAX_FILE_BYTES: "50000000" REPORT_PUBLISH_REQUIRED: ${{ github.event_name == 'schedule' || inputs.profile == 'release' }} steps: @@ -825,7 +822,9 @@ jobs: with: ref: ${{ github.sha }} path: .artifacts/performance-publisher - sparse-checkout: scripts/lib/kova-report-publish-files.mjs + sparse-checkout: | + scripts/lib/kova-report-publish-files.mjs + scripts/lib/kova-report-selector.mjs sparse-checkout-cone-mode: false fetch-depth: 1 persist-credentials: false @@ -913,12 +912,16 @@ jobs: exit 1 fi - mapfile -d '' report_jsons < <(find "$input_root" -type f -path "*/kova/reports/${LANE_ID}/*.json" -print0) - if [[ "${#report_jsons[@]}" != "1" ]]; then - echo "::${annotation}::Expected exactly one Kova JSON report for ${LANE_ID}; found ${#report_jsons[@]}." + mapfile -d '' report_dirs < <(find "$input_root" -type d -path "*/kova/reports/${LANE_ID}" -print0) + if [[ "${#report_dirs[@]}" != "1" ]]; then + echo "::${annotation}::Expected exactly one Kova report directory for ${LANE_ID}; found ${#report_dirs[@]}." exit 1 fi - report_json="$(realpath "${report_jsons[0]}")" + if ! report_json="$(node "$PERFORMANCE_REPORT_SELECTOR" --report-dir "${report_dirs[0]}")"; then + echo "::${annotation}::Exactly one full Kova JSON report is required for ${LANE_ID}." + exit 1 + fi + report_json="$(realpath "$report_json")" if [[ "$report_json" != "$input_root/"* || ! -s "$report_json" ]]; then echo "::${annotation}::Kova JSON report is outside the downloaded artifact or empty." exit 1 diff --git a/scripts/lib/kova-report-selector.mjs b/scripts/lib/kova-report-selector.mjs new file mode 100644 index 000000000000..8626aa16b5e1 --- /dev/null +++ b/scripts/lib/kova-report-selector.mjs @@ -0,0 +1,42 @@ +#!/usr/bin/env node + +import { readdirSync, realpathSync, statSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +function check(condition, message) { + if (!condition) { + throw new Error(message); + } +} + +export function selectKovaReport(reportDir) { + const root = realpathSync(reportDir); + const reports = readdirSync(root, { withFileTypes: true }) + .filter( + (entry) => + entry.isFile() && entry.name.endsWith(".json") && !entry.name.endsWith(".summary.json"), + ) + .map((entry) => path.join(root, entry.name)); + + check( + reports.length === 1, + `expected exactly one full Kova JSON report; found ${reports.length}`, + ); + check(statSync(reports[0]).size > 0, "full Kova JSON report is empty"); + return reports[0]; +} + +function runCli(argv) { + check(argv.length === 2 && argv[0] === "--report-dir", "usage: --report-dir "); + console.log(selectKovaReport(argv[1])); +} + +if (path.resolve(process.argv[1] ?? "") === fileURLToPath(import.meta.url)) { + try { + runCli(process.argv.slice(2)); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + } +} diff --git a/scripts/test-projects.test-support.mjs b/scripts/test-projects.test-support.mjs index a147305f5d59..02968ec5220a 100644 --- a/scripts/test-projects.test-support.mjs +++ b/scripts/test-projects.test-support.mjs @@ -1225,6 +1225,7 @@ const TOOLING_SOURCE_TEST_TARGETS = new Map([ ["scripts/lib/local-heavy-check-runtime.mjs", ["test/scripts/local-heavy-check-runtime.test.ts"]], ["scripts/lib/kova-report-gate.mjs", ["test/scripts/kova-report-gate.test.ts"]], ["scripts/lib/kova-report-publish-files.mjs", ["test/scripts/kova-report-publish-files.test.ts"]], + ["scripts/lib/kova-report-selector.mjs", ["test/scripts/kova-report-selector.test.ts"]], ["scripts/lib/kova-workflow-evidence.mjs", ["test/scripts/kova-workflow-evidence.test.ts"]], ["scripts/lib/managed-child-process.mjs", ["test/scripts/managed-child-process.test.ts"]], [ diff --git a/test/scripts/kova-report-selector.test.ts b/test/scripts/kova-report-selector.test.ts new file mode 100644 index 000000000000..ca1c7158ffba --- /dev/null +++ b/test/scripts/kova-report-selector.test.ts @@ -0,0 +1,63 @@ +import { spawnSync } from "node:child_process"; +import { mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { selectKovaReport } from "../../scripts/lib/kova-report-selector.mjs"; +import { useAutoCleanupTempDirTracker } from "../helpers/temp-dir.js"; + +const tempRoots = useAutoCleanupTempDirTracker(afterEach); +const SCRIPT_PATH = "scripts/lib/kova-report-selector.mjs"; + +function reportDir() { + const root = tempRoots.make("openclaw-kova-report-selector-"); + const dir = join(root, "reports"); + mkdirSync(dir); + return dir; +} + +describe("Kova report selector", () => { + it("selects the full report when Kova also writes its summary", () => { + const dir = reportDir(); + const report = join(dir, "kova-run-release.json"); + writeFileSync(report, '{"schemaVersion":"kova.report.v1"}\n'); + writeFileSync( + join(dir, "kova-run-release.summary.json"), + '{"schemaVersion":"kova.report.summary.v1"}\n', + ); + + expect(selectKovaReport(dir)).toBe(report); + const cli = spawnSync(process.execPath, [SCRIPT_PATH, "--report-dir", dir], { + encoding: "utf8", + }); + expect(cli).toMatchObject({ status: 0, stderr: "", stdout: `${report}\n` }); + }); + + it("fails closed without a full report", () => { + const dir = reportDir(); + writeFileSync( + join(dir, "kova-run-release.summary.json"), + '{"schemaVersion":"kova.report.summary.v1"}\n', + ); + + expect(() => selectKovaReport(dir)).toThrow( + "expected exactly one full Kova JSON report; found 0", + ); + }); + + it("fails closed with multiple full reports", () => { + const dir = reportDir(); + writeFileSync(join(dir, "kova-run-a.json"), "{}\n"); + writeFileSync(join(dir, "kova-run-b.json"), "{}\n"); + + expect(() => selectKovaReport(dir)).toThrow( + "expected exactly one full Kova JSON report; found 2", + ); + }); + + it("rejects an empty full report", () => { + const dir = reportDir(); + writeFileSync(join(dir, "kova-run-release.json"), ""); + + expect(() => selectKovaReport(dir)).toThrow("full Kova JSON report is empty"); + }); +}); diff --git a/test/scripts/openclaw-performance-workflow.test.ts b/test/scripts/openclaw-performance-workflow.test.ts index 005a6bc5036c..3a1fb8b4d0a3 100644 --- a/test/scripts/openclaw-performance-workflow.test.ts +++ b/test/scripts/openclaw-performance-workflow.test.ts @@ -356,10 +356,14 @@ describe("OpenClaw performance workflow", () => { expect(publisher?.env?.PERFORMANCE_PUBLISHER_HELPER).toContain( "scripts/lib/kova-report-publish-files.mjs", ); + expect(publisher?.env?.PERFORMANCE_REPORT_SELECTOR).toContain( + "scripts/lib/kova-report-selector.mjs", + ); expect(helper.with).toMatchObject({ ref: "${{ github.sha }}", path: ".artifacts/performance-publisher", - "sparse-checkout": "scripts/lib/kova-report-publish-files.mjs", + "sparse-checkout": + "scripts/lib/kova-report-publish-files.mjs\nscripts/lib/kova-report-selector.mjs\n", "sparse-checkout-cone-mode": false, "persist-credentials": false, }); @@ -709,6 +713,20 @@ esac expect(run).toContain('--model "$PERFORMANCE_MODEL_ID"'); }); + it("selects exactly one full Kova report across producer and publisher paths", () => { + const runKova = findStep("Run Kova"); + const validate = findStep("Validate Kova evidence"); + const publish = findStep("Prepare clawgrit report commit", "publish"); + + expect(runKova.run).toContain('kova-report-selector.mjs" --report-dir "$REPORT_DIR"'); + expect(validate.run).toContain('kova-report-selector.mjs" --report-dir "$REPORT_DIR"'); + expect(publish.run).toContain( + 'node "$PERFORMANCE_REPORT_SELECTOR" --report-dir "${report_dirs[0]}"', + ); + expect(runKova.run).not.toContain("tail -n 1"); + expect(publish.run).not.toContain("report_jsons"); + }); + it("installs local workspace packages beside the OCM root tarball", () => { const configure = findStep("Configure OCM local workspace dependencies"); @@ -743,7 +761,7 @@ esac expect(validateEvidence.if).toContain("always()"); expect(validateEvidence.if).toContain("steps.lane.outputs.run == 'true'"); - expect(validateEvidence.run).toContain('"$REPORT_DIR" -maxdepth 1 -type f -name'); + expect(validateEvidence.run).toContain('kova-report-selector.mjs" --report-dir "$REPORT_DIR"'); expect(validateEvidence.run).toContain('"$BUNDLE_DIR/bundle.json"'); expect(validateEvidence.run).toContain('"$SUMMARY_DIR/${LANE_ID}.md"'); expect(validateEvidence.run).toContain("exit 1");