From 366d09bcc730dffd9ed0783c37df506ac0f35955 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 10 Jul 2026 16:59:10 +0100 Subject: [PATCH] fix(ci): harden release performance evidence (#103790) --- .github/workflows/openclaw-performance.yml | 45 +++++-- docs/ci.md | 6 +- scripts/lib/kova-report-publish-files.mjs | 114 ++++++++++++++++++ scripts/lib/kova-workflow-evidence.mjs | 43 ++++++- scripts/test-projects.test-support.mjs | 2 + .../scripts/kova-report-publish-files.test.ts | 59 +++++++++ test/scripts/kova-workflow-evidence.test.ts | 81 ++++++++++++- .../openclaw-performance-workflow.test.ts | 30 +++++ 8 files changed, 362 insertions(+), 18 deletions(-) create mode 100644 scripts/lib/kova-report-publish-files.mjs create mode 100644 test/scripts/kova-report-publish-files.test.ts diff --git a/.github/workflows/openclaw-performance.yml b/.github/workflows/openclaw-performance.yml index 292efe2ea594..ad04f96801fe 100644 --- a/.github/workflows/openclaw-performance.yml +++ b/.github/workflows/openclaw-performance.yml @@ -33,7 +33,7 @@ on: default: false type: boolean live_openai_candidate: - description: Run the live OpenAI GPT 5.5 agent-turn lane + description: Run the live OpenAI GPT 5.6 agent-turn lane required: false default: false type: boolean @@ -70,7 +70,7 @@ env: OCM_VERSION: v0.2.15 OCM_LINUX_X64_SHA256: b849b8de5d77e97e0df9319703254ae95e29d7f26a7552ea79bf173ff110ea0a KOVA_REPOSITORY: openclaw/Kova - PERFORMANCE_MODEL_ID: gpt-5.5 + PERFORMANCE_MODEL_ID: gpt-5.6 KOVA_SCENARIO_TIMEOUT_MS: "300000" jobs: @@ -144,7 +144,7 @@ jobs: include_filters: "scenario:fresh-install,scenario:gateway-performance,scenario:agent-cold-warm-message" expected_release_entries: "fresh-install:fresh,fresh-install:onboarded-user,agent-cold-warm-message:mock-openai-provider,gateway-performance:many-bundled-plugins" - lane: live-openai-candidate - title: Kova live OpenAI GPT 5.5 agent turn + title: Kova live OpenAI GPT 5.6 agent turn auth: live repeat: "1" deep_profile: "false" @@ -352,7 +352,7 @@ jobs: echo "OPENCLAW_OCM_WORKSPACE_DEPENDENCY_DIRS=$workspace_dependency_dirs" } >> "$GITHUB_ENV" - - name: Pin Kova OpenAI model to GPT 5.5 + - name: Pin Kova OpenAI model to GPT 5.6 if: steps.lane.outputs.run == 'true' shell: bash run: | @@ -443,7 +443,7 @@ jobs: run: | set -euo pipefail if [[ -z "${OPENAI_API_KEY:-}" ]]; then - echo "OPENAI_API_KEY is not configured; live GPT 5.5 lane cannot run without live evidence." >> "$GITHUB_STEP_SUMMARY" + echo "OPENAI_API_KEY is not configured; live GPT 5.6 lane cannot run without live evidence." >> "$GITHUB_STEP_SUMMARY" exit 1 fi kova setup --ci --json @@ -502,7 +502,8 @@ jobs: --target "local-build:${GITHUB_WORKSPACE}" \ --repeat "$repeat" \ --include "$INCLUDE_FILTERS" \ - --auth "$AUTH_MODE" + --auth "$AUTH_MODE" \ + --model "$PERFORMANCE_MODEL_ID" effective_status="$status" if [[ "$FAIL_ON_REGRESSION" == "true" && "$status" != "0" ]]; then @@ -770,6 +771,8 @@ jobs: title: live OpenAI candidate env: LANE_ID: ${{ matrix.lane }} + PERFORMANCE_PUBLISHER_HELPER: ${{ github.workspace }}/.artifacts/performance-publisher/scripts/lib/kova-report-publish-files.mjs + PUBLISHED_REPORT_MAX_FILE_BYTES: "50000000" REPORT_PUBLISH_REQUIRED: ${{ github.event_name == 'schedule' || inputs.profile == 'release' }} steps: - name: Decide report publication lane @@ -786,6 +789,17 @@ jobs: fi echo "run=$run_lane" >> "$GITHUB_OUTPUT" + - name: Checkout performance publisher helper + if: steps.lane.outputs.run == 'true' + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + ref: ${{ github.sha }} + path: .artifacts/performance-publisher + sparse-checkout: scripts/lib/kova-report-publish-files.mjs + sparse-checkout-cone-mode: false + fetch-depth: 1 + persist-credentials: false + - name: Resolve Kova artifact id: artifact if: steps.lane.outputs.run == 'true' @@ -847,6 +861,7 @@ jobs: if: steps.download.outcome == 'success' continue-on-error: ${{ env.REPORT_PUBLISH_REQUIRED != 'true' }} env: + ARTIFACT_ID: ${{ steps.artifact.outputs.id }} INPUT_ROOT: ${{ steps.paths.outputs.input_root }} REPORTS_ROOT: ${{ steps.paths.outputs.reports_root }} TESTED_REF: ${{ needs.resolve_target.outputs.tested_ref }} @@ -863,7 +878,7 @@ jobs: echo "::${annotation}::Publisher paths escaped RUNNER_TEMP." exit 1 fi - if [[ ! "$PRODUCER_ATTEMPT" =~ ^[0-9]+$ || "$PRODUCER_ATTEMPT" -gt "$GITHUB_RUN_ATTEMPT" ]]; then + if [[ ! "$ARTIFACT_ID" =~ ^[0-9]+$ || ! "$PRODUCER_ATTEMPT" =~ ^[0-9]+$ || "$PRODUCER_ATTEMPT" -gt "$GITHUB_RUN_ATTEMPT" ]]; then echo "::${annotation}::Kova artifact attempt metadata is invalid." exit 1 fi @@ -958,8 +973,6 @@ jobs: install -m 0644 "$report_md" "$dest/report.md" fi install -m 0644 "$summary" "$dest/index.md" - mkdir -p "$dest/bundles" - cp -R "$bundle"/. "$dest/bundles/" if [[ -n "$source_dir" ]]; then mkdir -p "$dest/source" cp -R "$source_dir"/. "$dest/source/" @@ -973,6 +986,14 @@ jobs: fi fi + artifact_url="https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts/${ARTIFACT_ID}" + { + echo + echo "## Full diagnostic artifact" + echo + echo "The complete Kova bundle remains in [Actions artifact ${ARTIFACT_ID}](${artifact_url}); its checksum is published under the bundles directory." + } >> "$dest/index.md" + jq -n \ --arg repository "$GITHUB_REPOSITORY" \ --arg ref "$TESTED_REF" \ @@ -987,6 +1008,12 @@ jobs: '{repository:$repository,ref:$ref,sha:$sha,tested_ref:$ref,tested_sha:$sha,workflow_ref:$workflow_ref,workflow_sha:$workflow_sha,workflow:$workflow,run_id:$run_id,run_attempt:$run_attempt,lane:$lane,path:$path}' \ > "${reports_root}/openclaw-performance/${ref_slug}/latest-${LANE_ID}.json" + node "$PERFORMANCE_PUBLISHER_HELPER" \ + --bundle-dir "$bundle" \ + --bundle-destination "$dest/bundles" \ + --publish-root "$dest" \ + --max-file-bytes "$PUBLISHED_REPORT_MAX_FILE_BYTES" + git -C "$reports_root" config user.name "openclaw-performance[bot]" git -C "$reports_root" config user.email "openclaw-performance[bot]@users.noreply.github.com" git -C "$reports_root" add -- openclaw-performance diff --git a/docs/ci.md b/docs/ci.md index 652c7f5f4327..45c430d2410f 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -50,7 +50,7 @@ dispatch. | `ios-build` | Xcode project generation plus the iOS app simulator build | iOS app, shared app kit, or Swabble changes | | `android` | Android unit tests for both flavors plus one debug APK build | Android-relevant changes | | `test-performance-agent` | Separate workflow: daily Codex slow-test optimization after trusted activity | Main CI success or manual dispatch | -| `openclaw-performance` | Separate workflow: daily/on-demand Kova runtime performance reports with mock-provider, deep-profile, and GPT 5.5 live lanes | Scheduled and manual dispatch | +| `openclaw-performance` | Separate workflow: daily/on-demand Kova runtime performance reports with mock-provider, deep-profile, and GPT 5.6 live lanes | Scheduled and manual dispatch | ## Fail-fast order @@ -230,11 +230,11 @@ The workflow installs OCM from a pinned release and Kova from `openclaw/Kova` at - `mock-provider`: Kova diagnostic scenarios against a local-build runtime with deterministic fake OpenAI-compatible auth. - `mock-deep-profile`: CPU/heap/trace profiling for startup, gateway, and agent-turn hotspots. Runs on schedule, or on dispatch with `deep_profile=true`. -- `live-openai-candidate`: a real OpenAI `openai/gpt-5.5` agent turn, skipped when `OPENAI_API_KEY` is unavailable. Runs on schedule, or on dispatch with `live_openai_candidate=true`. +- `live-openai-candidate`: a real OpenAI `openai/gpt-5.6` agent turn, skipped when `OPENAI_API_KEY` is unavailable. Runs on schedule, or on dispatch with `live_openai_candidate=true`. The mock-provider lane also runs OpenClaw-native source probes after the Kova pass: gateway boot timing and memory across default, skipped-channel, internal-hook, and fifty-plugin startup cases; bundled plugin import RSS, repeated mock-OpenAI `channel-chat-baseline` hello loops, CLI startup commands against the booted gateway, and the SQLite state smoke performance probe. When the previous published mock-provider source report is available for the tested ref, the source summary compares current RSS and heap values against that baseline and marks large RSS increases as `watch`. The source probe Markdown summary lives at `source/index.md` in the report bundle, with raw JSON beside it. -Every lane uploads GitHub artifacts. A separate publisher job downloads and validates those artifacts, then mints a short-lived ClawSweeper GitHub App token scoped only to `openclaw/clawgrit-reports` contents and passes it only to the Git push step. It commits `report.json`, `report.md`, bundles, `index.md`, and source-probe artifacts under `openclaw-performance//-//`; the current tested-ref pointer is `openclaw-performance//latest-.json`. Scheduled runs and `profile=release` dispatches fail if app-token creation or report publication fails. Manual non-release dispatches keep publication advisory and retain the GitHub artifacts when authentication or publishing fails. The previous source baseline is fetched anonymously from the public reports repository, so a successful baseline fetch does not prove publisher authentication. +Every lane uploads its complete GitHub artifact, including CPU, heap, trace, and compressed diagnostic bundles. A separate publisher job downloads and validates those artifacts, then mints a short-lived ClawSweeper GitHub App token scoped only to `openclaw/clawgrit-reports` contents and passes it only to the Git push step. It commits `report.json`, `report.md`, `index.md`, source-probe artifacts, and bundle metadata/checksums under `openclaw-performance//-//`; the full diagnostic archive stays in the linked Actions artifact. The publisher rejects any report file over 50 MB before attempting a push. The current tested-ref pointer is `openclaw-performance//latest-.json`. Scheduled runs and `profile=release` dispatches fail if app-token creation or report publication fails. Manual non-release dispatches keep publication advisory and retain the GitHub artifacts when authentication or publishing fails. The previous source baseline is fetched anonymously from the public reports repository, so a successful baseline fetch does not prove publisher authentication. ## Full Release Validation diff --git a/scripts/lib/kova-report-publish-files.mjs b/scripts/lib/kova-report-publish-files.mjs new file mode 100644 index 000000000000..248a923f2f55 --- /dev/null +++ b/scripts/lib/kova-report-publish-files.mjs @@ -0,0 +1,114 @@ +#!/usr/bin/env node + +import { chmodSync, copyFileSync, mkdirSync, readdirSync, statSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +function check(condition, message) { + if (!condition) { + throw new Error(message); + } +} + +function positiveInteger(value, label) { + const parsed = Number(value); + check(Number.isSafeInteger(parsed) && parsed > 0, `${label} must be a positive integer`); + return parsed; +} + +function regularFiles(root) { + const files = []; + const visit = (dir, prefix) => { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const relative = path.join(prefix, entry.name); + const absolute = path.join(dir, entry.name); + if (entry.isDirectory()) { + visit(absolute, relative); + continue; + } + check(entry.isFile(), `publish tree contains non-regular file: ${relative}`); + files.push({ absolute, relative }); + } + }; + visit(root, ""); + return files; +} + +export function copyBundleMetadata({ bundleDir, destinationDir }) { + const entries = readdirSync(bundleDir, { withFileTypes: true }); + const metadata = entries + .filter( + (entry) => entry.isFile() && (entry.name === "bundle.json" || entry.name.endsWith(".sha256")), + ) + .map((entry) => entry.name) + .toSorted(); + check(metadata.includes("bundle.json"), "Kova bundle metadata is missing bundle.json"); + check( + metadata.some((name) => name.endsWith(".sha256")), + "Kova bundle metadata is missing a checksum", + ); + + mkdirSync(destinationDir, { recursive: true }); + for (const name of metadata) { + const source = path.join(bundleDir, name); + const destination = path.join(destinationDir, name); + copyFileSync(source, destination); + chmodSync(destination, 0o644); + } + return metadata; +} + +export function assertPublishedFileSizeLimit({ publishRoot, maxFileBytes }) { + const limit = positiveInteger(maxFileBytes, "max file bytes"); + const files = regularFiles(publishRoot); + for (const file of files) { + const size = statSync(file.absolute).size; + check( + size <= limit, + `refusing to publish oversized file ${file.relative}: ${size} bytes exceeds ${limit}`, + ); + } + return files.length; +} + +function parseArgs(argv) { + const values = {}; + for (let index = 0; index < argv.length; index += 2) { + const flag = argv[index]; + const value = argv[index + 1]; + check(flag?.startsWith("--") && value, "invalid CLI arguments"); + const key = flag.slice(2); + check( + ["bundle-dir", "bundle-destination", "publish-root", "max-file-bytes"].includes(key), + `unknown --${key}`, + ); + check(values[key] === undefined, `duplicate --${key}`); + values[key] = value; + } + for (const key of ["bundle-dir", "bundle-destination", "publish-root", "max-file-bytes"]) { + check(values[key], `missing --${key}`); + } + return values; +} + +function runCli() { + const args = parseArgs(process.argv.slice(2)); + const bundleFiles = copyBundleMetadata({ + bundleDir: args["bundle-dir"], + destinationDir: args["bundle-destination"], + }); + const publishedFileCount = assertPublishedFileSizeLimit({ + publishRoot: args["publish-root"], + maxFileBytes: args["max-file-bytes"], + }); + console.log(JSON.stringify({ bundleFiles, publishedFileCount })); +} + +if (path.resolve(process.argv[1] ?? "") === fileURLToPath(import.meta.url)) { + try { + runCli(); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + } +} diff --git a/scripts/lib/kova-workflow-evidence.mjs b/scripts/lib/kova-workflow-evidence.mjs index 9e37a31a3448..c328aabb3f56 100644 --- a/scripts/lib/kova-workflow-evidence.mjs +++ b/scripts/lib/kova-workflow-evidence.mjs @@ -3,7 +3,16 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; const AUTH_MODES = new Set(["live", "mock"]); -const CLI_KEYS = new Set(["auth", "include", "plan", "profile", "repeat", "report", "target"]); +const CLI_KEYS = new Set([ + "auth", + "include", + "model", + "plan", + "profile", + "repeat", + "report", + "target", +]); function check(condition, reason) { if (!condition) { @@ -55,7 +64,7 @@ function repeatIndexesFor(selectedPairs) { return new Map([...selectedPairs].map((key) => [key, new Set()])); } -function validateLiveRecord(record, key) { +function validateLiveRecord(record, key, expectedModel) { const auth = object(record.auth, `record ${displayPair(key)} auth`); const provider = object(record.providerEvidence, `record ${displayPair(key)} provider evidence`); check( @@ -75,7 +84,28 @@ function validateLiveRecord(record, key) { provider.available === true, `live record ${displayPair(key)} provider evidence was unavailable`, ); - positiveInteger(provider.requestCount, `live record ${displayPair(key)} provider request count`); + const requestCount = positiveInteger( + provider.requestCount, + `live record ${displayPair(key)} provider request count`, + ); + const models = array(provider.models, `live record ${displayPair(key)} provider models`); + check( + models.length === 1, + `live record ${displayPair(key)} provider model evidence was not exact`, + ); + const model = object(models[0], `live record ${displayPair(key)} provider model`); + check( + text(model.value, `live record ${displayPair(key)} provider model value`) === expectedModel, + `live record ${displayPair(key)} provider model did not match ${expectedModel}`, + ); + const modelCount = positiveInteger( + model.count, + `live record ${displayPair(key)} provider model count`, + ); + check( + modelCount === requestCount, + `live record ${displayPair(key)} provider model count did not match request count`, + ); } export function validateKovaWorkflowEvidence({ @@ -86,6 +116,7 @@ export function validateKovaWorkflowEvidence({ repeat, includeFilters, authMode, + expectedModel, }) { const expectedProfile = text(profile, "expected profile"); const expectedTarget = text(target, "expected target"); @@ -94,6 +125,7 @@ export function validateKovaWorkflowEvidence({ text(value, `expected include filter ${index}`), ); const expectedAuth = text(authMode, "expected auth mode"); + const expectedProviderModel = text(expectedModel, "expected model"); check(AUTH_MODES.has(expectedAuth), `unsupported expected auth mode ${expectedAuth}`); check(expectedFilters.length > 0, "expected include filters were empty"); @@ -180,7 +212,7 @@ export function validateKovaWorkflowEvidence({ const recordAuth = object(record.auth, `record ${displayPair(key)} auth`); check(recordAuth.mode === expectedAuth, `record ${displayPair(key)} auth mode did not match`); if (expectedAuth === "live") { - validateLiveRecord(record, key); + validateLiveRecord(record, key, expectedProviderModel); } } @@ -217,7 +249,7 @@ function parseCliArgs(argv) { check(!Object.hasOwn(flags, key), `duplicate --${key}`); flags[key] = value; } - for (const key of ["plan", "report", "profile", "target", "repeat", "include", "auth"]) { + for (const key of ["plan", "report", "profile", "target", "repeat", "include", "auth", "model"]) { text(flags[key], `--${key}`); } return flags; @@ -248,6 +280,7 @@ function runCli() { .map((value) => value.trim()) .filter(Boolean), authMode: flags.auth, + expectedModel: flags.model, }); console.log( `Kova plan/report evidence validated: ${result.pairCount} scenario/state pairs x ${result.repeat} repeats (${result.authMode})`, diff --git a/scripts/test-projects.test-support.mjs b/scripts/test-projects.test-support.mjs index 57288bbac36d..9bd6b102d07c 100644 --- a/scripts/test-projects.test-support.mjs +++ b/scripts/test-projects.test-support.mjs @@ -1201,6 +1201,8 @@ const TOOLING_SOURCE_TEST_TARGETS = new Map([ ["scripts/lib/live-docker-stage.sh", ["test/scripts/live-docker-stage.test.ts"]], ["scripts/lib/local-heavy-check-runtime.mjs", ["test/scripts/local-heavy-check-runtime.test.ts"]], ["scripts/lib/kova-report-gate.mjs", ["test/scripts/kova-report-gate.test.ts"]], + ["scripts/lib/kova-report-publish-files.mjs", ["test/scripts/kova-report-publish-files.test.ts"]], + ["scripts/lib/kova-workflow-evidence.mjs", ["test/scripts/kova-workflow-evidence.test.ts"]], ["scripts/lib/managed-child-process.mjs", ["test/scripts/managed-child-process.test.ts"]], [ "scripts/lib/windows-taskkill.mjs", diff --git a/test/scripts/kova-report-publish-files.test.ts b/test/scripts/kova-report-publish-files.test.ts new file mode 100644 index 000000000000..d1a067e49fdc --- /dev/null +++ b/test/scripts/kova-report-publish-files.test.ts @@ -0,0 +1,59 @@ +import { mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { + assertPublishedFileSizeLimit, + copyBundleMetadata, +} from "../../scripts/lib/kova-report-publish-files.mjs"; +import { useAutoCleanupTempDirTracker } from "../helpers/temp-dir.js"; + +const tempRoots = useAutoCleanupTempDirTracker(afterEach); + +describe("Kova report publish files", () => { + it("publishes bundle metadata while leaving the full archive in the Actions artifact", () => { + const root = tempRoots.make("openclaw-kova-publish-"); + const bundleDir = join(root, "artifact", "bundle"); + const destinationDir = join(root, "report", "bundles"); + mkdirSync(bundleDir, { recursive: true }); + writeFileSync(join(bundleDir, "bundle.json"), '{"schemaVersion":"kova.bundle.v1"}\n'); + writeFileSync(join(bundleDir, "bundle.tar.gz"), Buffer.alloc(200)); + writeFileSync(join(bundleDir, "bundle.tar.gz.sha256"), "abc bundle.tar.gz\n"); + + expect(copyBundleMetadata({ bundleDir, destinationDir })).toEqual([ + "bundle.json", + "bundle.tar.gz.sha256", + ]); + expect(readdirSync(destinationDir).sort()).toEqual(["bundle.json", "bundle.tar.gz.sha256"]); + expect(readFileSync(join(destinationDir, "bundle.tar.gz.sha256"), "utf8")).toBe( + "abc bundle.tar.gz\n", + ); + }); + + it("requires bundle metadata and a checksum", () => { + const root = tempRoots.make("openclaw-kova-publish-"); + const bundleDir = join(root, "bundle"); + mkdirSync(bundleDir); + writeFileSync(join(bundleDir, "bundle.tar.gz"), Buffer.alloc(1)); + + expect(() => + copyBundleMetadata({ bundleDir, destinationDir: join(root, "destination") }), + ).toThrow("Kova bundle metadata is missing bundle.json"); + + writeFileSync(join(bundleDir, "bundle.json"), "{}\n"); + expect(() => + copyBundleMetadata({ bundleDir, destinationDir: join(root, "destination") }), + ).toThrow("Kova bundle metadata is missing a checksum"); + }); + + it("accepts the size boundary and rejects the first oversized published file", () => { + const root = tempRoots.make("openclaw-kova-publish-"); + writeFileSync(join(root, "at-limit.json"), Buffer.alloc(100)); + + expect(assertPublishedFileSizeLimit({ publishRoot: root, maxFileBytes: 100 })).toBe(1); + + writeFileSync(join(root, "oversized.json"), Buffer.alloc(101)); + expect(() => assertPublishedFileSizeLimit({ publishRoot: root, maxFileBytes: 100 })).toThrow( + "oversized.json: 101 bytes exceeds 100", + ); + }); +}); diff --git a/test/scripts/kova-workflow-evidence.test.ts b/test/scripts/kova-workflow-evidence.test.ts index 5ec2163eeaf1..b5d30e914e11 100644 --- a/test/scripts/kova-workflow-evidence.test.ts +++ b/test/scripts/kova-workflow-evidence.test.ts @@ -13,6 +13,7 @@ type Pair = { const PROFILE = "release"; const TARGET = "local-build:/work/openclaw"; +const MODEL = "gpt-5.6"; const INCLUDE_FILTERS = ["scenario:scenario-a", "scenario:scenario-b"]; const FIRST_PAIR: Pair = { scenario: "scenario-a", state: "state-a" }; const PAIRS: Pair[] = [FIRST_PAIR, { scenario: "scenario-b", state: "state-b" }]; @@ -60,6 +61,7 @@ function record(pair: Pair, repeatIndex: number, repeat: number, authMode: "live available: true, environmentDependent: true, requestCount: 1, + models: [{ value: MODEL, count: 1 }], source: "openclaw-timeline", } : { @@ -107,7 +109,12 @@ function report({ function validate( lanePlan: JsonObject, laneReport: JsonObject, - options: { authMode?: "live" | "mock"; includeFilters?: string[]; repeat?: number } = {}, + options: { + authMode?: "live" | "mock"; + expectedModel?: string; + includeFilters?: string[]; + repeat?: number; + } = {}, ) { return validateKovaWorkflowEvidence({ plan: lanePlan, @@ -117,6 +124,7 @@ function validate( repeat: options.repeat ?? 2, includeFilters: options.includeFilters ?? INCLUDE_FILTERS, authMode: options.authMode ?? "mock", + expectedModel: options.expectedModel ?? MODEL, }); } @@ -162,6 +170,8 @@ function runCli({ repeat, "--include", INCLUDE_FILTERS.join(","), + "--model", + MODEL, ]; if (includeAuth) { args.push("--auth", "mock"); @@ -246,6 +256,75 @@ describe("Kova workflow evidence", () => { }); }); + it("rejects live evidence for a different provider model", () => { + const liveFilters = ["scenario:scenario-a"]; + const lanePlan = plan([FIRST_PAIR], 1, liveFilters); + const laneReport = report({ + authMode: "live", + includeFilters: liveFilters, + pairs: [FIRST_PAIR], + repeat: 1, + }); + (firstRecordOf(laneReport).providerEvidence as JsonObject).models = [ + { value: "gpt-5.5", count: 1 }, + ]; + + expect(() => + validate(lanePlan, laneReport, { + authMode: "live", + includeFilters: liveFilters, + repeat: 1, + }), + ).toThrow("live record scenario-a/state-a provider model did not match gpt-5.6"); + }); + + it("rejects mixed live provider models", () => { + const liveFilters = ["scenario:scenario-a"]; + const lanePlan = plan([FIRST_PAIR], 1, liveFilters); + const laneReport = report({ + authMode: "live", + includeFilters: liveFilters, + pairs: [FIRST_PAIR], + repeat: 1, + }); + const providerEvidence = firstRecordOf(laneReport).providerEvidence as JsonObject; + providerEvidence.requestCount = 2; + providerEvidence.models = [ + { value: MODEL, count: 1 }, + { value: "gpt-5.5", count: 1 }, + ]; + + expect(() => + validate(lanePlan, laneReport, { + authMode: "live", + includeFilters: liveFilters, + repeat: 1, + }), + ).toThrow("live record scenario-a/state-a provider model evidence was not exact"); + }); + + it("rejects live provider model count drift", () => { + const liveFilters = ["scenario:scenario-a"]; + const lanePlan = plan([FIRST_PAIR], 1, liveFilters); + const laneReport = report({ + authMode: "live", + includeFilters: liveFilters, + pairs: [FIRST_PAIR], + repeat: 1, + }); + (firstRecordOf(laneReport).providerEvidence as JsonObject).models = [ + { value: MODEL, count: 2 }, + ]; + + expect(() => + validate(lanePlan, laneReport, { + authMode: "live", + includeFilters: liveFilters, + repeat: 1, + }), + ).toThrow("live record scenario-a/state-a provider model count did not match request count"); + }); + it("rejects plan and report schema drift", () => { const badPlan = plan(); badPlan.schemaVersion = "kova.matrix.plan.v0"; diff --git a/test/scripts/openclaw-performance-workflow.test.ts b/test/scripts/openclaw-performance-workflow.test.ts index 75b78fc47736..45fa4abab731 100644 --- a/test/scripts/openclaw-performance-workflow.test.ts +++ b/test/scripts/openclaw-performance-workflow.test.ts @@ -96,6 +96,8 @@ describe("OpenClaw performance workflow", () => { expect( installRun.indexOf('npm --prefix "$KOVA_SRC" ci --ignore-scripts --no-audit --no-fund'), ).toBeLessThan(installRun.indexOf('cat > "$HOME/.local/bin/kova"')); + expect(workflow).toContain("PERFORMANCE_MODEL_ID: gpt-5.6"); + expect(workflow).toContain("Kova live OpenAI GPT 5.6 agent turn"); }); it("resolves each target once before benchmark and publication fan out", () => { @@ -319,6 +321,33 @@ describe("OpenClaw performance workflow", () => { expect(publish.run).not.toContain("rebase FETCH_HEAD"); }); + it("publishes bounded bundle metadata while retaining full diagnostics as an artifact", () => { + const workflow = readWorkflow(); + const publisher = workflow.jobs?.publish; + const helper = findStep("Checkout performance publisher helper", "publish"); + const prepare = findStep("Prepare clawgrit report commit", "publish"); + const upload = findStep("Upload Kova artifacts"); + + expect(publisher?.env?.PUBLISHED_REPORT_MAX_FILE_BYTES).toBe("50000000"); + expect(publisher?.env?.PERFORMANCE_PUBLISHER_HELPER).toContain( + "scripts/lib/kova-report-publish-files.mjs", + ); + expect(helper.with).toMatchObject({ + ref: "${{ github.sha }}", + path: ".artifacts/performance-publisher", + "sparse-checkout": "scripts/lib/kova-report-publish-files.mjs", + "sparse-checkout-cone-mode": false, + "persist-credentials": false, + }); + expect(upload.with?.path).toContain(".artifacts/kova/bundles/${{ matrix.lane }}"); + expect(prepare.env?.ARTIFACT_ID).toBe("${{ steps.artifact.outputs.id }}"); + expect(prepare.run).toContain('node "$PERFORMANCE_PUBLISHER_HELPER"'); + expect(prepare.run).toContain('--bundle-destination "$dest/bundles"'); + expect(prepare.run).toContain('--max-file-bytes "$PUBLISHED_REPORT_MAX_FILE_BYTES"'); + expect(prepare.run).toContain("The complete Kova bundle remains in [Actions artifact"); + expect(prepare.run).not.toContain('cp -R "$bundle"/. "$dest/bundles/"'); + }); + it("reuses the producing artifact when only publisher jobs rerun", () => { const artifact = findStep("Resolve Kova artifact", "publish"); const root = mkdtempSync(join(realpathSync(tmpdir()), "openclaw-artifact-resolver-")); @@ -653,6 +682,7 @@ esac expect(run).toContain('--repeat "$repeat"'); expect(run).toContain('--include "$INCLUDE_FILTERS"'); expect(run).toContain('--auth "$AUTH_MODE"'); + expect(run).toContain('--model "$PERFORMANCE_MODEL_ID"'); }); it("installs local workspace packages beside the OCM root tarball", () => {