diff --git a/.github/workflows/openclaw-performance.yml b/.github/workflows/openclaw-performance.yml index 3906c4c440ed..13a3765c3813 100644 --- a/.github/workflows/openclaw-performance.yml +++ b/.github/workflows/openclaw-performance.yml @@ -52,6 +52,11 @@ on: required: false default: "" type: string + kova_config_contract: + description: Optional fixture-contract override for a custom Kova ref + required: false + default: "" + type: string dispatch_id: description: Optional parent workflow dispatch identifier required: false @@ -70,8 +75,8 @@ env: OCM_VERSION: v0.2.29 OCM_LINUX_X64_SHA256: d966098d6ba2bc10891be3c76e162a37b07f28c4f51da75d2eb509886eb7e1cf KOVA_REPOSITORY: openclaw/Kova - KOVA_CANONICAL_CONFIG_REF: 283070760a16655b28835061774158b8b11b4aff - KOVA_LEGACY_LIST_CONFIG_REF: f3d037b5b8aacd6adf8ef1dd2ea4c1d778ec7c6c + KOVA_CANONICAL_CONFIG_REF: 0f9e678e239b45db46d2bd930b7983203580df78 + KOVA_LEGACY_LIST_CONFIG_REF: 0f9e678e239b45db46d2bd930b7983203580df78 PERFORMANCE_MODEL_ID: gpt-5.6-luna # Release matrices cold-build the candidate runtime before measurement. KOVA_SCENARIO_TIMEOUT_MS: ${{ inputs.profile == 'release' && '900000' || '300000' }} @@ -86,6 +91,8 @@ jobs: tested_ref: ${{ steps.resolve.outputs.tested_ref }} tested_sha: ${{ steps.resolve.outputs.tested_sha }} kova_ref: ${{ steps.resolve.outputs.kova_ref }} + kova_config_contract: ${{ steps.resolve.outputs.kova_config_contract }} + kova_ref_trusted_for_live: ${{ steps.resolve.outputs.kova_ref_trusted_for_live }} steps: - name: Resolve OpenClaw target ref id: resolve @@ -93,6 +100,7 @@ jobs: GH_TOKEN: ${{ github.token }} TARGET_REF_INPUT: ${{ inputs.target_ref }} KOVA_REF_INPUT: ${{ inputs.kova_ref }} + KOVA_CONFIG_CONTRACT_INPUT: ${{ inputs.kova_config_contract }} shell: bash run: | set -euo pipefail @@ -118,17 +126,43 @@ jobs: fi kova_ref="${KOVA_REF_INPUT:-}" - if [[ -z "$kova_ref" ]]; then - schema_content="$({ + kova_config_contract="${KOVA_CONFIG_CONTRACT_INPUT:-}" + if [[ "$kova_ref" == *$'\n'* || "$kova_ref" == *$'\r'* ]]; then + echo "::error::kova_ref must be a single line." + exit 1 + fi + if [[ "$kova_config_contract" == *$'\n'* || "$kova_config_contract" == *$'\r'* ]]; then + echo "::error::kova_config_contract must be a single line." + exit 1 + fi + + if [[ -z "$kova_ref" || -z "$kova_config_contract" ]]; then + if schema_content="$({ gh api "repos/${GITHUB_REPOSITORY}/contents/src/config/zod-schema.agent-defaults.ts?ref=${resolved_sha}" --jq '.content' | base64 --decode - })" - if grep -Fqx ' mediaModels: z' <<<"$schema_content"; then - kova_ref="$KOVA_CANONICAL_CONFIG_REF" - elif grep -Fqx ' imageGenerationModel: AgentToolModelSchema.optional(),' <<<"$schema_content"; then - kova_ref="$KOVA_LEGACY_LIST_CONFIG_REF" + })"; then + : + elif [[ -z "$kova_ref" ]]; then + echo "::error::Unable to inspect the Kova config-fixture contract for target ${resolved_sha}. Supply kova_ref explicitly and optionally set kova_config_contract for that producer." + exit 1 else - echo "::error::Target ${resolved_sha} has no recognized Kova config-fixture contract. Supply kova_ref explicitly after validating the fixture." + schema_content="" + fi + if [[ -n "$schema_content" ]] && grep -Fqx ' mediaModels: z' <<<"$schema_content"; then + detected_kova_config_contract="canonical" + default_kova_ref="$KOVA_CANONICAL_CONFIG_REF" + elif [[ -n "$schema_content" ]] && grep -Fqx ' imageGenerationModel: AgentToolModelSchema.optional(),' <<<"$schema_content"; then + detected_kova_config_contract="legacy-list" + default_kova_ref="$KOVA_LEGACY_LIST_CONFIG_REF" + else + detected_kova_config_contract="" + default_kova_ref="" + fi + if [[ -n "$detected_kova_config_contract" ]]; then + kova_config_contract="${kova_config_contract:-$detected_kova_config_contract}" + kova_ref="${kova_ref:-$default_kova_ref}" + elif [[ -z "$kova_ref" ]]; then + echo "::error::Target ${resolved_sha} has no recognized Kova config-fixture contract. Supply kova_ref explicitly and optionally set kova_config_contract for that producer." exit 1 fi fi @@ -137,6 +171,12 @@ jobs: echo "tested_ref=$tested_ref" >> "$GITHUB_OUTPUT" echo "tested_sha=$resolved_sha" >> "$GITHUB_OUTPUT" echo "kova_ref=$kova_ref" >> "$GITHUB_OUTPUT" + echo "kova_config_contract=$kova_config_contract" >> "$GITHUB_OUTPUT" + if [[ "$kova_ref" == "$KOVA_CANONICAL_CONFIG_REF" || "$kova_ref" == "$KOVA_LEGACY_LIST_CONFIG_REF" ]]; then + echo "kova_ref_trusted_for_live=true" >> "$GITHUB_OUTPUT" + else + echo "kova_ref_trusted_for_live=false" >> "$GITHUB_OUTPUT" + fi kova: name: ${{ matrix.title }} @@ -176,6 +216,7 @@ jobs: expected_release_entries: "agent-cold-warm-message:mock-openai-provider" env: KOVA_REF: ${{ needs.resolve_target.outputs.kova_ref }} + KOVA_OPENCLAW_CONFIG_CONTRACT: ${{ needs.resolve_target.outputs.kova_config_contract }} KOVA_HOME: ${{ github.workspace }}/.artifacts/kova/home/${{ matrix.lane }} PERFORMANCE_HELPER_DIR: ${{ github.workspace }}/.artifacts/performance-workflow REPORT_DIR: ${{ github.workspace }}/.artifacts/kova/reports/${{ matrix.lane }} @@ -193,6 +234,7 @@ jobs: MATRIX_REPEAT: ${{ matrix.repeat }} MATRIX_DEEP_PROFILE: ${{ matrix.deep_profile }} MATRIX_LIVE: ${{ matrix.live }} + KOVA_REF_TRUSTED_FOR_LIVE: ${{ needs.resolve_target.outputs.kova_ref_trusted_for_live }} steps: - name: Decide lane id: lane @@ -209,6 +251,10 @@ jobs: run_lane=false reason="live_openai_candidate input is false" fi + if [[ "$LANE_ID" == "live-openai-candidate" && "$run_lane" == "true" && "$KOVA_REF_TRUSTED_FOR_LIVE" != "true" ]]; then + echo "::error::The live OpenAI lane only executes a reviewed immutable Kova default. Omit kova_ref or update the pinned workflow defaults after review." + exit 1 + fi echo "run=$run_lane" >> "$GITHUB_OUTPUT" if [[ "$run_lane" != "true" ]]; then echo "Skipping ${LANE_ID}: ${reason}" >> "$GITHUB_STEP_SUMMARY" @@ -437,8 +483,8 @@ jobs: id: kova if: steps.lane.outputs.run == 'true' env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL }} + OPENAI_API_KEY: ${{ matrix.live == 'true' && secrets.OPENAI_API_KEY || '' }} + OPENAI_BASE_URL: ${{ matrix.live == 'true' && secrets.OPENAI_BASE_URL || '' }} shell: bash run: | set -euo pipefail @@ -497,7 +543,11 @@ jobs: effective_status="$status" if [[ "$evidence_status" == "0" && "$FAIL_ON_REGRESSION" == "true" && "$status" != "0" ]]; then - if node "$PERFORMANCE_HELPER_DIR/scripts/lib/kova-report-gate.mjs" "$report_json" + gate_args=("$report_json") + if [[ "$KOVA_REF" == "$KOVA_CANONICAL_CONFIG_REF" || "$KOVA_REF" == "$KOVA_LEGACY_LIST_CONFIG_REF" ]]; then + gate_args+=(--require-instrumented-performance-contract) + fi + if node "$PERFORMANCE_HELPER_DIR/scripts/lib/kova-report-gate.mjs" "${gate_args[@]}" then effective_status=0 { diff --git a/scripts/lib/kova-report-gate.d.mts b/scripts/lib/kova-report-gate.d.mts index 26c76debc430..d92da0544394 100644 --- a/scripts/lib/kova-report-gate.d.mts +++ b/scripts/lib/kova-report-gate.d.mts @@ -1,4 +1,11 @@ -export function evaluateToleratedPartialKovaReport(report: unknown): +export interface KovaReportGateOptions { + requireInstrumentedPerformanceContract?: boolean; +} + +export function evaluateToleratedPartialKovaReport( + report: unknown, + options?: KovaReportGateOptions, +): | { ok: boolean; reason?: undefined; @@ -7,7 +14,10 @@ export function evaluateToleratedPartialKovaReport(report: unknown): ok: boolean; reason: string; }; -export function evaluateToleratedProfiledKovaReport(report: unknown): +export function evaluateToleratedProfiledKovaReport( + report: unknown, + options?: KovaReportGateOptions, +): | { ok: boolean; reason?: undefined; @@ -16,7 +26,10 @@ export function evaluateToleratedProfiledKovaReport(report: unknown): ok: boolean; reason: string; }; -export function evaluateToleratedKovaReport(report: unknown): +export function evaluateToleratedKovaReport( + report: unknown, + options?: KovaReportGateOptions, +): | { ok: boolean; classification: string; diff --git a/scripts/lib/kova-report-gate.mjs b/scripts/lib/kova-report-gate.mjs index 5eb7ff018e8a..e980442b2b9d 100644 --- a/scripts/lib/kova-report-gate.mjs +++ b/scripts/lib/kova-report-gate.mjs @@ -11,6 +11,9 @@ const SCHEMA = { }; const PROFILED_INTERPRETATION = "instrumented run; CPU/RSS can include profiler and diagnostic overhead"; +const INSTRUMENTED_PERFORMANCE_REASON = "instrumented-performance-measurement"; +const REQUIRE_INSTRUMENTED_PERFORMANCE_CONTRACT_FLAG = + "--require-instrumented-performance-contract"; const RSS_METRICS = ["peakRssMb", "resourcePeakGatewayRssMb"]; const CPU_METRICS = ["cpuPercentMax"]; const DIRECT_VIOLATIONS = new Set(["cpuPercentMax", "peakRssMb"]); @@ -27,6 +30,15 @@ function object(value, label) { return value; } +function hasOwn(value, key) { + return ( + value !== null && + typeof value === "object" && + !Array.isArray(value) && + Object.hasOwn(value, key) + ); +} + function array(value, label) { check(Array.isArray(value), `invalid ${label}`); return value; @@ -54,6 +66,10 @@ function finite(value, label) { return value; } +function finiteOrNull(value) { + return value === null || (typeof value === "number" && Number.isFinite(value)); +} + function stateId(record) { return text(object(record.state, "record state").id, "record state id"); } @@ -351,9 +367,149 @@ function validatePerformance(report, records, repeat) { return groupsByKey; } -function validateGateCards(gate) { +function instrumentedRecordKey(record, required) { + const assessment = object( + record.performanceThresholdAssessment, + "performance threshold assessment", + ); + const skipped = array(assessment.skipped, "skipped performance thresholds"); + const skippedCount = count(assessment.skippedCount, "skipped threshold count"); + const complete = skippedCount === 0; + check( + record.profiling.enabled === true && + record.profiling.affectsPerformanceMeasurements === true && + record.profiling.baselineEligible === false && + record.measurements.profilingAffectsPerformanceMeasurements === true && + record.measurements.performanceThresholdSkippedCount === skippedCount && + assessment.schemaVersion === "kova.performanceThresholdAssessment.v1" && + assessment.complete === complete && + (complete + ? assessment.reason === null && assessment.rerun === null + : assessment.reason === INSTRUMENTED_PERFORMANCE_REASON && + typeof assessment.rerun === "string") && + skipped.length === skippedCount, + "instrumented performance assessment was invalid", + ); + if (complete) { + return null; + } + check( + skipped.every( + (entry) => + entry?.status === "SKIPPED" && + entry.reason === INSTRUMENTED_PERFORMANCE_REASON && + entry.affectsRecordStatus === false && + typeof entry.metric === "string" && + entry.metric.trim().length > 0 && + finiteOrNull(entry.actual) && + finiteOrNull(entry.threshold), + ), + "skipped performance threshold was invalid", + ); + const first = skipped[0]; + return JSON.stringify([ + text(record.scenario, "record scenario"), + stateId(record), + required, + skippedCount, + first.metric, + first.actual, + first.threshold, + ]); +} + +function recordAffectsPerformanceMeasurements(record) { + const profiling = object(record.profiling, "record profiling"); + const measurements = object(record.measurements, "record measurements"); + const affects = + profiling.nodeProfile === true || + profiling.heapSnapshot === true || + profiling.diagnosticReport === true; + check( + profiling.affectsPerformanceMeasurements === affects && + measurements.profilingAffectsPerformanceMeasurements === affects, + "record profiling performance provenance drift", + ); + return affects; +} + +function expectedInstrumentedEvidence(gate, records) { + const warnings = array(gate.warning, "gate warning policy"); + const evidence = new Map(); + for (const record of records) { + const affectsPerformanceMeasurements = recordAffectsPerformanceMeasurements(record); + if (!affectsPerformanceMeasurements || record.status !== "PASS") { + continue; + } + const scenario = text(record.scenario, "record scenario"); + const state = stateId(record); + const required = !warnings.some( + (entry) => entry?.scenario === scenario && (!entry.state || entry.state === state), + ); + const key = instrumentedRecordKey(record, required); + if (key === null) { + continue; + } + check( + gate.verdict === "PARTIAL", + "non-PARTIAL report contained incomplete instrumented performance evidence", + ); + evidence.set(key, (evidence.get(key) ?? 0) + 1); + } + return evidence; +} + +function instrumentedCardKey(card) { + check( + card.status === "SKIPPED" && + typeof card.required === "boolean" && + array(card.violations, "instrumented performance card violations").length === 0, + "instrumented performance card metadata was invalid", + ); + const scenario = text(card.scenario, "instrumented performance card scenario"); + const state = text(card.state, "instrumented performance card state"); + const measurements = object(card.measurements, "instrumented performance card measurements"); + return JSON.stringify([ + scenario, + state, + card.required, + count(measurements.skippedCount, "card skipped threshold count", { positive: true }), + text(measurements.firstMetric, "card first skipped metric"), + finiteOrNull(measurements.firstActual) ? measurements.firstActual : "invalid", + finiteOrNull(measurements.firstThreshold) ? measurements.firstThreshold : "invalid", + ]); +} + +function hasInstrumentedPerformanceContractMarker(gate, records) { + if ( + hasOwn(gate, "instrumentedPerformanceIncompleteCount") || + (Array.isArray(gate.cards) && + gate.cards.some( + (card) => hasOwn(card, "kind") && card.kind === "instrumented-performance-thresholds", + )) + ) { + return true; + } + return records.some((record) => { + if (record === null || typeof record !== "object" || Array.isArray(record)) { + return false; + } + return ( + hasOwn(record, "performanceThresholdAssessment") || + hasOwn(record.profiling, "affectsPerformanceMeasurements") || + hasOwn(record.measurements, "profilingAffectsPerformanceMeasurements") || + hasOwn(record.measurements, "performanceThresholdSkippedCount") + ); + }); +} + +function validateGateCards(gate, records, validateInstrumentedPerformance) { const cards = array(gate.cards, "gate cards"); const severities = { blocking: 0, warning: 0, info: 0 }; + const instrumentedEvidence = validateInstrumentedPerformance + ? expectedInstrumentedEvidence(gate, records) + : new Map(); + let requiredInstrumented = 0; for (const cardValue of cards) { const card = object(cardValue, "gate card"); const severity = card.severity; @@ -367,13 +523,36 @@ function validateGateCards(gate) { ); } if (severity === "warning") { - check( - card.kind === "missing-required-coverage" && card.status === "MISSING", - "unexpected warning gate card", - ); + if (card.kind === "instrumented-performance-thresholds") { + const key = instrumentedCardKey(card); + const remaining = instrumentedEvidence.get(key) ?? 0; + check(remaining > 0, "instrumented performance card lacked matching evidence"); + if (remaining === 1) { + instrumentedEvidence.delete(key); + } else { + instrumentedEvidence.set(key, remaining - 1); + } + requiredInstrumented += Number(card.required); + } else { + check( + card.kind === "missing-required-coverage" && card.status === "MISSING", + "unexpected warning gate card", + ); + } } severities[severity] += 1; } + check( + instrumentedEvidence.size === 0, + "instrumented performance evidence lacked matching gate cards", + ); + if (validateInstrumentedPerformance) { + check( + count(gate.instrumentedPerformanceIncompleteCount, "instrumented incomplete count") === + requiredInstrumented, + "instrumented incomplete count drift", + ); + } check( count(gate.blockingCount, "blocking count") === severities.blocking, "blocking count drift", @@ -384,10 +563,10 @@ function validateGateCards(gate) { count(gate.missingRequiredCount, "missing required count") === severities.info, "missing count drift", ); - return cards; + return { cards, requiredInstrumented }; } -function validateEnvelope(reportValue) { +function validateEnvelope(reportValue, options = {}) { const report = object(reportValue, "report"); const gate = object(report.gate, "gate"); const controls = object(report.controls, "controls"); @@ -414,14 +593,22 @@ function validateEnvelope(reportValue) { gate.partial === true && gate.complete === false && gate.ok === false, "gate metadata was not partial", ); - const records = array(report.records, "records").map(validateRecord); + const recordValues = array(report.records, "records"); + const validateInstrumentedPerformance = + options.requireInstrumentedPerformanceContract === true || + hasInstrumentedPerformanceContractMarker(gate, recordValues); + const records = recordValues.map(validateRecord); check(records.length > 0, "report had no records"); - const cards = validateGateCards(gate); validateBaselines(report, gate); validateSummary(report, records); validateTargetCleanup(report); const groups = validatePerformance(report, records, repeat); - return { report, gate, records, cards, groups }; + const { cards, requiredInstrumented } = validateGateCards( + gate, + records, + validateInstrumentedPerformance, + ); + return { report, gate, records, cards, groups, requiredInstrumented }; } function deepProfiled(record) { @@ -513,11 +700,15 @@ function evaluate(evaluator) { } } -export function evaluateToleratedPartialKovaReport(report) { +export function evaluateToleratedPartialKovaReport(report, options = {}) { return evaluate(() => { - const { gate, records, cards } = validateEnvelope(report); + const { gate, records, cards, requiredInstrumented } = validateEnvelope(report, options); check(gate.verdict === "PARTIAL", "gate verdict was not PARTIAL"); check(gate.blockingCount === 0, "PARTIAL gate had blocking cards"); + check( + requiredInstrumented === 0, + "PARTIAL gate had incomplete required instrumented performance evidence", + ); check( cards.every((card) => card.severity !== "blocking"), "PARTIAL gate had a blocking card", @@ -533,9 +724,9 @@ export function evaluateToleratedPartialKovaReport(report) { }); } -export function evaluateToleratedProfiledKovaReport(report) { +export function evaluateToleratedProfiledKovaReport(report, options = {}) { return evaluate(() => { - const { gate, records, cards, groups } = validateEnvelope(report); + const { gate, records, cards, groups } = validateEnvelope(report, options); check(gate.verdict === "DO_NOT_SHIP", "gate verdict was not DO_NOT_SHIP"); check( records.every((record) => record.status === "PASS" || record.status === "FAIL"), @@ -566,24 +757,42 @@ export function evaluateToleratedProfiledKovaReport(report) { }); } -export function evaluateToleratedKovaReport(report) { - const partial = evaluateToleratedPartialKovaReport(report); +export function evaluateToleratedKovaReport(report, options = {}) { + const partial = evaluateToleratedPartialKovaReport(report, options); if (partial.ok) { return { ok: true, classification: "filtered-partial" }; } - const profiled = evaluateToleratedProfiledKovaReport(report); + const profiled = evaluateToleratedProfiledKovaReport(report, options); if (profiled.ok) { return { ok: true, classification: "profiled-resource-only" }; } return { ok: false, reason: `partial: ${partial.reason}; profiled: ${profiled.reason}` }; } -function readCliReportPath() { - const reportPath = process.argv[2] || process.env.REPORT_JSON; - if (!reportPath) { - throw new Error("usage: node scripts/lib/kova-report-gate.mjs "); +function readCliInvocation() { + let reportPath; + let requireInstrumentedPerformanceContract = false; + for (const arg of process.argv.slice(2)) { + if (arg === REQUIRE_INSTRUMENTED_PERFORMANCE_CONTRACT_FLAG) { + requireInstrumentedPerformanceContract = true; + } else if (arg.startsWith("-")) { + throw new Error(`unknown option: ${arg}`); + } else if (reportPath === undefined) { + reportPath = arg; + } else { + throw new Error(`unexpected argument: ${arg}`); + } } - return reportPath; + reportPath ??= process.env.REPORT_JSON; + if (!reportPath) { + throw new Error( + `usage: node scripts/lib/kova-report-gate.mjs [${REQUIRE_INSTRUMENTED_PERFORMANCE_CONTRACT_FLAG}] `, + ); + } + return { + options: { requireInstrumentedPerformanceContract }, + reportPath, + }; } const modulePath = fs.realpathSync.native(fileURLToPath(import.meta.url)); @@ -591,8 +800,9 @@ const invokedPath = process.argv[1] ? fs.realpathSync.native(path.resolve(proces if (modulePath === invokedPath) { try { - const report = JSON.parse(fs.readFileSync(readCliReportPath(), "utf8")); - const result = evaluateToleratedKovaReport(report); + const invocation = readCliInvocation(); + const report = JSON.parse(fs.readFileSync(invocation.reportPath, "utf8")); + const result = evaluateToleratedKovaReport(report, invocation.options); if (!result.ok) { console.error(`Kova verdict is not tolerable: ${result.reason}`); process.exit(1); diff --git a/test/scripts/kova-report-gate.test.ts b/test/scripts/kova-report-gate.test.ts index 5949dc288a69..a72c748eb700 100644 --- a/test/scripts/kova-report-gate.test.ts +++ b/test/scripts/kova-report-gate.test.ts @@ -26,6 +26,11 @@ const STATE = "mock-openai-provider"; const SURFACE = "agent-cli-local-turn"; const PROFILED_INTERPRETATION = "instrumented run; CPU/RSS can include profiler and diagnostic overhead"; +const INSTRUMENTED_PERFORMANCE_INTERPRETATION = + "instrumented diagnostic run; CPU, RSS, and latency can include profiler overhead"; +const STRICT_INSTRUMENTED_PERFORMANCE_OPTIONS = { + requireInstrumentedPerformanceContract: true, +}; function objectAt(value: unknown): JsonObject { if (value === null || typeof value !== "object" || Array.isArray(value)) { @@ -113,6 +118,7 @@ function targetCleanup() { function normalProfiling() { return { + affectsPerformanceMeasurements: false, affectsResourceMeasurements: false, baselineEligible: true, deepProfile: false, @@ -128,6 +134,7 @@ function normalProfiling() { function deepProfiling() { return { + affectsPerformanceMeasurements: true, affectsResourceMeasurements: true, baselineEligible: false, deepProfile: true, @@ -167,11 +174,14 @@ function partialReport(): JsonObject { complete: false, enabled: true, infoCount: 1, + instrumentedPerformanceIncompleteCount: 0, missingRequiredCount: 1, ok: false, partial: true, + required: [], schemaVersion: "kova.gate.v1", verdict: "PARTIAL", + warning: [], warningCount: 0, }, mode: "execution", @@ -205,6 +215,7 @@ function partialReport(): JsonObject { measurements: { cpuPercentMax: 80, peakRssMb: 650, + profilingAffectsPerformanceMeasurements: false, }, phases: [ { @@ -260,11 +271,14 @@ function profiledResourceReport(): JsonObject { complete: false, enabled: true, infoCount: 1, + instrumentedPerformanceIncompleteCount: 0, missingRequiredCount: 1, ok: false, partial: true, + required: [], schemaVersion: "kova.gate.v1", verdict: "DO_NOT_SHIP", + warning: [], warningCount: 0, }, mode: "execution", @@ -298,6 +312,7 @@ function profiledResourceReport(): JsonObject { measurements: { cpuPercentMax: 156.2, peakRssMb: 923.7, + profilingAffectsPerformanceMeasurements: true, profilingAffectsResourceMeasurements: true, profilingBaselineEligible: false, profilingEnabled: true, @@ -350,6 +365,8 @@ function attachPassingBaseline(report: JsonObject): void { baselineEntryCount: 1, generatedAt: "2026-07-09T00:00:00.000Z", groups: [], + instrumentedPerformanceGroupCount: 0, + instrumentedPerformanceGroups: [], missing: [], missingBaselineCount: 0, ok: true, @@ -363,6 +380,8 @@ function attachPassingBaseline(report: JsonObject): void { baselineEntryCount: 1, missing: [], missingBaselineCount: 0, + instrumentedPerformanceGroupCount: 0, + instrumentedPerformanceGroups: [], ok: true, regressedGroups: [], regressionCount: 0, @@ -370,6 +389,155 @@ function attachPassingBaseline(report: JsonObject): void { }; } +function markRecordInstrumented(report: JsonObject, recordIndex = 0): JsonObject { + const record = objectAt(valueAt(report, ["records", recordIndex])); + const measurements = objectAt(record.measurements); + record.profiling = { + ...deepProfiling(), + affectsPerformanceMeasurements: true, + interpretation: INSTRUMENTED_PERFORMANCE_INTERPRETATION, + }; + measurements.profilingAffectsPerformanceMeasurements = true; + const performance = objectAt(report.performance); + const group = objectAt(arrayAt(performance.groups)[0]); + performance.profiledRunCount = Number(performance.profiledRunCount) + 1; + group.profiledRunCount = Number(group.profiledRunCount) + 1; + setAt(report, ["performance", "groups", 0, "resourceInterpretation"], "instrumented"); + return record; +} + +function attachCompleteInstrumentedAssessment(report: JsonObject): void { + const record = markRecordInstrumented(report); + setCompleteInstrumentedAssessment(record); +} + +function setCompleteInstrumentedAssessment(record: JsonObject): void { + const measurements = objectAt(record.measurements); + measurements.performanceThresholdSkippedCount = 0; + record.performanceThresholdAssessment = { + complete: true, + reason: null, + rerun: null, + schemaVersion: "kova.performanceThresholdAssessment.v1", + skipped: [], + skippedCount: 0, + }; +} + +function attachInstrumentedPerformanceWarning( + report: JsonObject, + recordIndex = 0, + required = true, +): void { + const metricId = "resourceByRole.status-cli.peakRssMb"; + const actual = 612.4; + const threshold = 900; + const record = markRecordInstrumented(report, recordIndex); + const measurements = objectAt(record.measurements); + measurements.performanceThresholdSkippedCount = 1; + record.performanceThresholdAssessment = { + complete: false, + reason: "instrumented-performance-measurement", + rerun: "rerun without profiling for gateable performance evidence", + schemaVersion: "kova.performanceThresholdAssessment.v1", + skipped: [ + { + actual, + affectsRecordStatus: false, + measurementMetric: "peakRssMb", + message: `${metricId} was not adjudicated because the run was instrumented`, + metric: metricId, + observedOverThreshold: false, + reason: "instrumented-performance-measurement", + role: "status-cli", + status: "SKIPPED", + threshold, + }, + ], + skippedCount: 1, + }; + const gate = objectAt(report.gate); + if (!required) { + arrayAt(gate.warning).push({ scenario: SCENARIO, state: STATE }); + } + arrayAt(gate.cards).push({ + actual: `${metricId} ${actual}`, + expected: `${metricId} <= ${threshold}`, + failedCommand: null, + impact: + "This run can reject functional failures, but it cannot approve the release until the scenario is rerun without profiling.", + kind: "instrumented-performance-thresholds", + likelyOwner: "Kova", + measurements: { + firstActual: actual, + firstMetric: metricId, + firstThreshold: threshold, + skippedCount: 1, + }, + required, + scenario: SCENARIO, + severity: "warning", + state: STATE, + status: "SKIPPED", + summary: + "1 performance threshold(s) were not adjudicated because profiling can distort CPU, RSS, and latency.", + title: "Instrumented Performance Evidence", + violations: [], + }); + gate.warningCount = Number(gate.warningCount) + 1; + gate.instrumentedPerformanceIncompleteCount = + Number(gate.instrumentedPerformanceIncompleteCount) + Number(required); +} + +function eraseInstrumentedPerformanceEvidence(report: JsonObject): void { + const record = objectAt(arrayAt(report.records)[0]); + const measurements = objectAt(record.measurements); + delete record.performanceThresholdAssessment; + delete measurements.performanceThresholdSkippedCount; + const gate = objectAt(report.gate); + gate.cards = arrayAt(gate.cards).filter( + (card) => objectAt(card).kind !== "instrumented-performance-thresholds", + ); + gate.warningCount = 0; + gate.instrumentedPerformanceIncompleteCount = 0; +} + +function stripInstrumentedPerformanceContract(report: JsonObject): JsonObject { + const gate = objectAt(report.gate); + delete gate.instrumentedPerformanceIncompleteCount; + gate.cards = arrayAt(gate.cards).filter( + (card) => objectAt(card).kind !== "instrumented-performance-thresholds", + ); + for (const recordValue of arrayAt(report.records)) { + const record = objectAt(recordValue); + const measurements = objectAt(record.measurements); + const profiling = objectAt(record.profiling); + delete profiling.affectsPerformanceMeasurements; + delete measurements.profilingAffectsPerformanceMeasurements; + delete measurements.performanceThresholdSkippedCount; + delete record.performanceThresholdAssessment; + } + return report; +} + +function duplicatePassingRecord(report: JsonObject): void { + const records = arrayAt(report.records); + records.push(structuredClone(records[0])); + const controls = objectAt(report.controls); + controls.repeat = 2; + report.summary = { statuses: { PASS: 2 }, total: 2 }; + const performance = objectAt(report.performance); + performance.repeat = 2; + const group = objectAt(arrayAt(performance.groups)[0]); + group.sampleCount = 2; + group.statuses = { PASS: 2 }; + for (const metricValue of Object.values(objectAt(group.metrics))) { + const metricObject = objectAt(metricValue); + metricObject.count = 2; + metricObject.samples = [arrayAt(metricObject.samples)[0], arrayAt(metricObject.samples)[0]]; + } +} + function blockingCard(report: JsonObject): JsonObject { const cards = arrayAt(objectAt(report.gate).cards); const card = cards.find((candidate) => objectAt(candidate).severity === "blocking"); @@ -429,6 +597,86 @@ describe("scripts/lib/kova-report-gate.mjs", () => { }); }); + it("accepts a historical filtered PARTIAL v1 report only in automatic mode", () => { + const report = stripInstrumentedPerformanceContract(partialReport()); + + expect(evaluateToleratedKovaReport(report)).toEqual({ + classification: "filtered-partial", + ok: true, + }); + expect(evaluateToleratedKovaReport(report, STRICT_INSTRUMENTED_PERFORMANCE_OPTIONS).ok).toBe( + false, + ); + }); + + it("accepts a historical profiled resource-only v1 report only in automatic mode", () => { + const report = stripInstrumentedPerformanceContract(profiledResourceReport()); + + expect(evaluateToleratedKovaReport(report)).toEqual({ + classification: "profiled-resource-only", + ok: true, + }); + expect(evaluateToleratedKovaReport(report, STRICT_INSTRUMENTED_PERFORMANCE_OPTIONS).ok).toBe( + false, + ); + }); + + it("rejects partially present instrumented performance contract markers", () => { + const report = stripInstrumentedPerformanceContract(partialReport()); + setAt(report, ["gate", "instrumentedPerformanceIncompleteCount"], 0); + + expectPartialRejection(report); + }); + + it("rejects incomplete required instrumented evidence on a filtered PARTIAL report", () => { + const report = partialReport(); + attachInstrumentedPerformanceWarning(report); + + expect(evaluateToleratedPartialKovaReport(report)).toEqual({ + ok: false, + reason: "PARTIAL gate had incomplete required instrumented performance evidence", + }); + expect(evaluateToleratedKovaReport(report).ok).toBe(false); + }); + + it("accepts an explicit complete assessment when no thresholds were skipped", () => { + const report = partialReport(); + attachCompleteInstrumentedAssessment(report); + + expect(evaluateToleratedPartialKovaReport(report)).toEqual({ ok: true }); + }); + + it("reconciles repeated instrumented warnings one-to-one", () => { + const report = partialReport(); + duplicatePassingRecord(report); + attachInstrumentedPerformanceWarning(report, 0, false); + attachInstrumentedPerformanceWarning(report, 1, false); + + expect(evaluateToleratedPartialKovaReport(report)).toEqual({ ok: true }); + }); + + it("accepts nullable instrumented threshold values", () => { + const report = partialReport(); + attachInstrumentedPerformanceWarning(report, 0, false); + setAt(report, ["records", 0, "performanceThresholdAssessment", "skipped", 0, "actual"], null); + setAt( + report, + ["records", 0, "performanceThresholdAssessment", "skipped", 0, "threshold"], + null, + ); + setAt(report, ["gate", "cards", 1, "measurements", "firstActual"], null); + setAt(report, ["gate", "cards", 1, "measurements", "firstThreshold"], null); + + expect(evaluateToleratedPartialKovaReport(report)).toEqual({ ok: true }); + }); + + it("derives advisory instrumented evidence from the gate warning policy", () => { + const report = partialReport(); + attachInstrumentedPerformanceWarning(report, 0, false); + + expect(evaluateToleratedPartialKovaReport(report)).toEqual({ ok: true }); + }); + it("accepts a declared collector-only phase without commands", () => { const report = partialReport(); setAt(report, ["records", 0, "phases", 0], { @@ -509,11 +757,18 @@ describe("scripts/lib/kova-report-gate.mjs", () => { it("accepts omitted violations on a profiled PASS record", () => { const report = profiledResourceReport(); - addProfiledPassRecord(report); + setCompleteInstrumentedAssessment(addProfiledPassRecord(report)); expect(evaluateToleratedProfiledKovaReport(report)).toEqual({ ok: true }); }); + it("rejects a profiled PASS record without a threshold assessment", () => { + const report = profiledResourceReport(); + addProfiledPassRecord(report); + + expectProfiledRejection(report); + }); + it.each(malformedViolationLists)( "rejects present non-array %s violations on a PARTIAL PASS record", (_label, violations) => { @@ -529,6 +784,7 @@ describe("scripts/lib/kova-report-gate.mjs", () => { (_label, violations) => { const report = profiledResourceReport(); const passRecord = addProfiledPassRecord(report); + setCompleteInstrumentedAssessment(passRecord); passRecord.violations = violations; expectProfiledRejection(report); @@ -538,6 +794,7 @@ describe("scripts/lib/kova-report-gate.mjs", () => { it("rejects hidden violations on PASS records", () => { const report = profiledResourceReport(); const passRecord = addProfiledPassRecord(report); + setCompleteInstrumentedAssessment(passRecord); passRecord.violations = [{ message: "hidden violation" }]; expectProfiledRejection(report); @@ -907,6 +1164,139 @@ describe("scripts/lib/kova-report-gate.mjs", () => { "rejects PARTIAL reports without sampled CPU", (report) => deleteAt(report, ["performance", "groups", 0, "metrics", "cpuPercentMax"]), ], + [ + "rejects instrumented warning count drift", + (report) => { + attachInstrumentedPerformanceWarning(report); + setAt(report, ["gate", "cards", 1, "measurements", "skippedCount"], 2); + }, + ], + [ + "rejects duplicate repeat cards that mask distinct assessments", + (report) => { + duplicatePassingRecord(report); + attachInstrumentedPerformanceWarning(report, 0); + attachInstrumentedPerformanceWarning(report, 1); + setAt( + report, + ["records", 1, "performanceThresholdAssessment", "skipped", 0, "metric"], + "cpuPercentMax", + ); + }, + ], + [ + "rejects instrumented warnings without profiler provenance", + (report) => { + attachInstrumentedPerformanceWarning(report); + setAt(report, ["records", 0, "profiling", "affectsPerformanceMeasurements"], false); + }, + ], + [ + "rejects erased instrumented evidence with false producer mirrors", + (report) => { + attachInstrumentedPerformanceWarning(report); + eraseInstrumentedPerformanceEvidence(report); + setAt(report, ["records", 0, "profiling", "affectsPerformanceMeasurements"], false); + setAt( + report, + ["records", 0, "measurements", "profilingAffectsPerformanceMeasurements"], + false, + ); + }, + ], + [ + "rejects erased instrumented evidence with missing producer mirrors", + (report) => { + attachInstrumentedPerformanceWarning(report); + eraseInstrumentedPerformanceEvidence(report); + deleteAt(report, ["records", 0, "profiling", "affectsPerformanceMeasurements"]); + deleteAt(report, ["records", 0, "measurements", "profilingAffectsPerformanceMeasurements"]); + }, + ], + [ + "rejects required instrumented cards marked advisory", + (report) => { + attachInstrumentedPerformanceWarning(report); + setAt(report, ["gate", "cards", 1, "required"], false); + setAt(report, ["gate", "instrumentedPerformanceIncompleteCount"], 0); + }, + ], + [ + "rejects instrumented incomplete count drift", + (report) => { + attachInstrumentedPerformanceWarning(report); + setAt(report, ["gate", "instrumentedPerformanceIncompleteCount"], 0); + }, + ], + [ + "rejects missing instrumented incomplete count", + (report) => { + attachInstrumentedPerformanceWarning(report); + deleteAt(report, ["gate", "instrumentedPerformanceIncompleteCount"]); + }, + ], + [ + "rejects instrumented warnings with status-affecting assessments", + (report) => { + attachInstrumentedPerformanceWarning(report); + setAt( + report, + ["records", 0, "performanceThresholdAssessment", "skipped", 0, "affectsRecordStatus"], + true, + ); + }, + ], + [ + "rejects unsupported instrumented baseline warning cards", + (report) => { + const gate = objectAt(report.gate); + arrayAt(gate.cards).push({ + failedCommand: null, + kind: "instrumented-performance-thresholds", + likelyOwner: "Kova", + measurements: { skippedMetrics: ["peakRssMb"] }, + required: true, + scenario: SCENARIO, + severity: "warning", + state: STATE, + status: "SKIPPED", + violations: [], + }); + gate.warningCount = 1; + gate.instrumentedPerformanceIncompleteCount = 1; + }, + ], + [ + "rejects instrumented assessments without matching warning cards", + (report) => { + attachInstrumentedPerformanceWarning(report); + arrayAt(objectAt(report.gate).cards).pop(); + setAt(report, ["gate", "warningCount"], 0); + }, + ], + [ + "rejects profiled records with erased threshold assessments", + (report) => { + attachInstrumentedPerformanceWarning(report); + deleteAt(report, ["records", 0, "performanceThresholdAssessment"]); + deleteAt(report, ["records", 0, "measurements", "performanceThresholdSkippedCount"]); + arrayAt(objectAt(report.gate).cards).pop(); + setAt(report, ["gate", "warningCount"], 0); + setAt(report, ["gate", "instrumentedPerformanceIncompleteCount"], 0); + }, + ], + [ + "rejects zeroed assessments that still claim incomplete evidence", + (report) => { + attachInstrumentedPerformanceWarning(report); + setAt(report, ["records", 0, "performanceThresholdAssessment", "skipped"], []); + setAt(report, ["records", 0, "performanceThresholdAssessment", "skippedCount"], 0); + setAt(report, ["records", 0, "measurements", "performanceThresholdSkippedCount"], 0); + arrayAt(objectAt(report.gate).cards).pop(); + setAt(report, ["gate", "warningCount"], 0); + setAt(report, ["gate", "instrumentedPerformanceIncompleteCount"], 0); + }, + ], [ "rejects PARTIAL one-sided baselines", (report) => { @@ -935,6 +1325,44 @@ describe("scripts/lib/kova-report-gate.mjs", () => { expect(result.stdout).toContain("profiled-resource-only"); }); + it("keeps required instrumented PARTIAL evidence non-zero at the CLI boundary", () => { + const report = partialReport(); + attachInstrumentedPerformanceWarning(report); + const result = spawnSync(process.execPath, [SCRIPT_PATH, writeReport(report)], { + cwd: process.cwd(), + encoding: "utf8", + }); + + expect(result.status).toBe(1); + expect(result.stderr).toContain( + "PARTIAL gate had incomplete required instrumented performance evidence", + ); + }); + + it("allows advisory instrumented PARTIAL evidence at the CLI boundary", () => { + const report = partialReport(); + attachInstrumentedPerformanceWarning(report, 0, false); + const result = spawnSync(process.execPath, [SCRIPT_PATH, writeReport(report)], { + cwd: process.cwd(), + encoding: "utf8", + }); + + expect(result.status).toBe(0); + expect(result.stdout).toContain("filtered-partial"); + }); + + it("requires the current producer contract when the CLI flag is present", () => { + const reportPath = writeReport(stripInstrumentedPerformanceContract(partialReport())); + const result = spawnSync( + process.execPath, + [SCRIPT_PATH, reportPath, "--require-instrumented-performance-contract"], + { cwd: process.cwd(), encoding: "utf8" }, + ); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("record profiling performance provenance drift"); + }); + it("exits non-zero for malformed tolerated-report candidates", () => { const report = partialReport(); setAt(report, ["summary", "total"], 2); diff --git a/test/scripts/openclaw-performance-workflow.test.ts b/test/scripts/openclaw-performance-workflow.test.ts index 8b4105f1a875..f7631d888b58 100644 --- a/test/scripts/openclaw-performance-workflow.test.ts +++ b/test/scripts/openclaw-performance-workflow.test.ts @@ -13,10 +13,12 @@ import { import { tmpdir } from "node:os"; import { join } from "node:path"; import { expectDefined } from "@openclaw/normalization-core"; -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it } from "vitest"; import { parse } from "yaml"; +import { useAutoCleanupTempDirTracker } from "../helpers/temp-dir.js"; const WORKFLOW = ".github/workflows/openclaw-performance.yml"; +const tempDirs = useAutoCleanupTempDirTracker(afterEach); type WorkflowStep = { name?: string; @@ -84,24 +86,67 @@ describe("OpenClaw performance workflow", () => { it("pins the Kova evaluator with release validation contracts", () => { const workflow = readFileSync(WORKFLOW, "utf8"); - const canonicalKovaRef = "283070760a16655b28835061774158b8b11b4aff"; - const legacyKovaRef = "f3d037b5b8aacd6adf8ef1dd2ea4c1d778ec7c6c"; + const canonicalKovaRef = "0f9e678e239b45db46d2bd930b7983203580df78"; + const legacyKovaRef = "0f9e678e239b45db46d2bd930b7983203580df78"; const install = findStep("Install OCM and Kova"); const installRun = install.run ?? ""; const resolveTarget = findStep("Resolve OpenClaw target ref", "resolve_target"); expect(workflow).toContain(`KOVA_CANONICAL_CONFIG_REF: ${canonicalKovaRef}`); expect(workflow).toContain(`KOVA_LEGACY_LIST_CONFIG_REF: ${legacyKovaRef}`); + expect(workflow).toContain("kova_config_contract:"); + expect(workflow).toContain("Optional fixture-contract override for a custom Kova ref"); expect(readWorkflow().jobs?.resolve_target?.outputs?.kova_ref).toBe( "${{ steps.resolve.outputs.kova_ref }}", ); + expect(readWorkflow().jobs?.resolve_target?.outputs?.kova_config_contract).toBe( + "${{ steps.resolve.outputs.kova_config_contract }}", + ); + expect(readWorkflow().jobs?.resolve_target?.outputs?.kova_ref_trusted_for_live).toBe( + "${{ steps.resolve.outputs.kova_ref_trusted_for_live }}", + ); expect(resolveTarget.env?.KOVA_REF_INPUT).toBe("${{ inputs.kova_ref }}"); + expect(resolveTarget.env?.KOVA_CONFIG_CONTRACT_INPUT).toBe( + "${{ inputs.kova_config_contract }}", + ); expect(resolveTarget.run).toContain("zod-schema.agent-defaults.ts?ref=${resolved_sha}"); expect(resolveTarget.run).toContain("KOVA_CANONICAL_CONFIG_REF"); expect(resolveTarget.run).toContain("KOVA_LEGACY_LIST_CONFIG_REF"); + expect(resolveTarget.run).toContain('detected_kova_config_contract="canonical"'); + expect(resolveTarget.run).toContain('detected_kova_config_contract="legacy-list"'); + expect(resolveTarget.run).toContain('kova_ref="${KOVA_REF_INPUT:-}"'); + expect(resolveTarget.run).toContain('kova_ref="${kova_ref:-$default_kova_ref}"'); + expect(resolveTarget.run).toContain( + 'if [[ -z "$kova_ref" || -z "$kova_config_contract" ]]; then', + ); + expect(resolveTarget.run).toContain('if schema_content="$({'); + expect(resolveTarget.run).toContain('elif [[ -z "$kova_ref" ]]; then'); + expect(resolveTarget.run).toContain('schema_content=""'); + expect(resolveTarget.run).toContain("Supply kova_ref explicitly"); + expect( + resolveTarget.run?.indexOf('if [[ -z "$kova_ref" || -z "$kova_config_contract" ]]; then'), + ).toBeLessThan(resolveTarget.run?.indexOf('schema_content="$({') ?? -1); + expect(resolveTarget.run).toContain( + 'echo "kova_config_contract=$kova_config_contract" >> "$GITHUB_OUTPUT"', + ); + expect(resolveTarget.run).toContain( + 'if [[ "$kova_ref" == "$KOVA_CANONICAL_CONFIG_REF" || "$kova_ref" == "$KOVA_LEGACY_LIST_CONFIG_REF" ]]; then', + ); + expect(resolveTarget.run).toContain( + 'echo "kova_ref_trusted_for_live=true" >> "$GITHUB_OUTPUT"', + ); + expect(resolveTarget.run).toContain( + 'echo "kova_ref_trusted_for_live=false" >> "$GITHUB_OUTPUT"', + ); expect(readWorkflow().jobs?.kova?.env?.KOVA_REF).toBe( "${{ needs.resolve_target.outputs.kova_ref }}", ); + expect(readWorkflow().jobs?.kova?.env?.KOVA_OPENCLAW_CONFIG_CONTRACT).toBe( + "${{ needs.resolve_target.outputs.kova_config_contract }}", + ); + expect(readWorkflow().jobs?.kova?.env?.KOVA_REF_TRUSTED_FOR_LIVE).toBe( + "${{ needs.resolve_target.outputs.kova_ref_trusted_for_live }}", + ); expect(installRun).toContain( 'npm --prefix "$KOVA_SRC" ci --ignore-scripts --no-audit --no-fund', ); @@ -121,6 +166,67 @@ describe("OpenClaw performance workflow", () => { expect(workflow).toContain("Kova live OpenAI GPT 5.6 agent turn"); }); + it("keeps live credentials away from custom Kova refs", () => { + const decideLane = findStep("Decide lane"); + const configureLiveAuth = findStep("Configure live OpenAI auth"); + const runKova = findStep("Run Kova"); + const root = mkdtempSync(join(realpathSync(tmpdir()), "openclaw-kova-live-ref-")); + const output = join(root, "output"); + const decideLaneRun = (decideLane.run ?? "") + .replaceAll("${{ github.event_name }}", "workflow_dispatch") + .replaceAll("${{ inputs.deep_profile || 'false' }}", "false") + .replaceAll("${{ inputs.live_openai_candidate || 'false' }}", "true"); + + expect(decideLane.run).toContain( + 'if [[ "$LANE_ID" == "live-openai-candidate" && "$run_lane" == "true" && "$KOVA_REF_TRUSTED_FOR_LIVE" != "true" ]]; then', + ); + expect(decideLane.run).toContain( + "The live OpenAI lane only executes a reviewed immutable Kova default.", + ); + expect(decideLane.run?.indexOf("KOVA_REF_TRUSTED_FOR_LIVE")).toBeLessThan( + decideLane.run?.indexOf('echo "run=$run_lane"') ?? -1, + ); + expect(configureLiveAuth.if).toBe( + "${{ steps.lane.outputs.run == 'true' && matrix.live == 'true' }}", + ); + expect(runKova.env?.OPENAI_API_KEY).toBe( + "${{ matrix.live == 'true' && secrets.OPENAI_API_KEY || '' }}", + ); + expect(runKova.env?.OPENAI_BASE_URL).toBe( + "${{ matrix.live == 'true' && secrets.OPENAI_BASE_URL || '' }}", + ); + + try { + const rejected = spawnSync("bash", ["-c", decideLaneRun], { + encoding: "utf8", + env: { + ...process.env, + GITHUB_OUTPUT: output, + KOVA_REF_TRUSTED_FOR_LIVE: "false", + LANE_ID: "live-openai-candidate", + }, + }); + expect(rejected.status).toBe(1); + expect(rejected.stdout).toContain( + "The live OpenAI lane only executes a reviewed immutable Kova default.", + ); + + const accepted = spawnSync("bash", ["-c", decideLaneRun], { + encoding: "utf8", + env: { + ...process.env, + GITHUB_OUTPUT: output, + KOVA_REF_TRUSTED_FOR_LIVE: "true", + LANE_ID: "live-openai-candidate", + }, + }); + expect(accepted.status).toBe(0); + expect(readFileSync(output, "utf8")).toContain("run=true\n"); + } finally { + rmSync(root, { force: true, recursive: true }); + } + }); + it("pins the OCM release archive and checksum", () => { const workflow = readFileSync(WORKFLOW, "utf8"); const installRun = findStep("Install OCM and Kova").run ?? ""; @@ -766,7 +872,7 @@ esac const runKova = findStep("Run Kova"); expect(runKova.run).toContain( - 'node "$PERFORMANCE_HELPER_DIR/scripts/lib/kova-report-gate.mjs" "$report_json"', + 'node "$PERFORMANCE_HELPER_DIR/scripts/lib/kova-report-gate.mjs" "${gate_args[@]}"', ); expect(runKova.run).not.toContain("report.summary?.statuses ?? {}"); expect(runKova.run).toContain( @@ -774,6 +880,71 @@ esac ); }); + it("preserves required PARTIAL failures and clears only advisory PARTIAL failures", () => { + const run = findStep("Run Kova").run ?? ""; + const startMarker = 'effective_status="$status"'; + const endMarker = 'echo "effective_status=$effective_status" >> "$GITHUB_OUTPUT"'; + const start = run.indexOf(startMarker); + const end = run.indexOf(endMarker, start); + expect(start).toBeGreaterThanOrEqual(0); + expect(end).toBeGreaterThan(start); + const gateScript = run.slice(start, end + endMarker.length); + const root = tempDirs.make("openclaw-kova-partial-gate-"); + const binDir = join(root, "bin"); + const fakeNode = join(binDir, "node"); + mkdirSync(binDir, { recursive: true }); + writeFileSync( + fakeNode, + [ + "#!/bin/sh", + 'printf "%s\\n" "$*" >> "$GATE_INVOCATIONS"', + '[ "$PARTIAL_POLICY" = "advisory" ]', + "", + ].join("\n"), + ); + chmodSync(fakeNode, 0o755); + + for (const [partialPolicy, expectedStatus] of [ + ["required", "17"], + ["advisory", "0"], + ] as const) { + const output = join(root, `${partialPolicy}.output`); + const summary = join(root, `${partialPolicy}.summary`); + const invocations = join(root, `${partialPolicy}.invocations`); + const result = spawnSync("bash", ["-c", gateScript], { + encoding: "utf8", + env: { + ...process.env, + evidence_status: "0", + FAIL_ON_REGRESSION: "true", + GATE_INVOCATIONS: invocations, + GITHUB_OUTPUT: output, + GITHUB_STEP_SUMMARY: summary, + KOVA_CANONICAL_CONFIG_REF: "trusted", + KOVA_LEGACY_LIST_CONFIG_REF: "trusted", + KOVA_REF: "trusted", + PARTIAL_POLICY: partialPolicy, + PATH: `${binDir}:${process.env.PATH ?? ""}`, + PERFORMANCE_HELPER_DIR: root, + report_json: join(root, `${partialPolicy}.json`), + status: "17", + }, + }); + expect(result.status).toBe(0); + expect(readFileSync(output, "utf8")).toBe(`effective_status=${expectedStatus}\n`); + expect(readFileSync(invocations, "utf8")).toContain( + "--require-instrumented-performance-contract", + ); + if (partialPolicy === "advisory") { + expect(readFileSync(summary, "utf8")).toContain( + "trusted report adapter found only filtered coverage", + ); + } else { + expect(existsSync(summary)).toBe(false); + } + } + }); + it("passes one comma-delimited include set to the lane plan and run", () => { const plan = findStep("Kova version and plan sanity"); const runKova = findStep("Run Kova"); @@ -894,6 +1065,22 @@ esac expect(run).toContain('--include "$INCLUDE_FILTERS"'); expect(run).toContain('--auth "$AUTH_MODE"'); expect(run).toContain('--model "$PERFORMANCE_MODEL_ID"'); + expect(run).toContain('gate_args=("$report_json")'); + expect(run).toContain( + 'if [[ "$KOVA_REF" == "$KOVA_CANONICAL_CONFIG_REF" || "$KOVA_REF" == "$KOVA_LEGACY_LIST_CONFIG_REF" ]]; then', + ); + expect(run).toContain("gate_args+=(--require-instrumented-performance-contract)"); + expect(run).toContain( + 'node "$PERFORMANCE_HELPER_DIR/scripts/lib/kova-report-gate.mjs" "${gate_args[@]}"', + ); + expect(run.indexOf('gate_args=("$report_json")')).toBeLessThan( + run.indexOf("gate_args+=(--require-instrumented-performance-contract)"), + ); + expect(run.indexOf("gate_args+=(--require-instrumented-performance-contract)")).toBeLessThan( + run.indexOf( + 'node "$PERFORMANCE_HELPER_DIR/scripts/lib/kova-report-gate.mjs" "${gate_args[@]}"', + ), + ); }); it("selects exactly one full Kova report across producer and publisher paths", () => { @@ -937,6 +1124,12 @@ esac expect(configureAuth.run).toContain("cannot run without live evidence"); expect(configureAuth.run).toContain("exit 1"); expect(configureAuth.run).not.toContain("will be skipped"); + expect(runKova.env?.OPENAI_API_KEY).toBe( + "${{ matrix.live == 'true' && secrets.OPENAI_API_KEY || '' }}", + ); + expect(runKova.env?.OPENAI_BASE_URL).toBe( + "${{ matrix.live == 'true' && secrets.OPENAI_BASE_URL || '' }}", + ); expect(runKova.run).not.toContain('echo "skipped=true" >> "$GITHUB_OUTPUT"'); });