diff --git a/.github/workflows/openclaw-performance.yml b/.github/workflows/openclaw-performance.yml index f79a4729340b..46e1e7cd6ab2 100644 --- a/.github/workflows/openclaw-performance.yml +++ b/.github/workflows/openclaw-performance.yml @@ -75,8 +75,9 @@ env: OCM_VERSION: v0.2.32 OCM_LINUX_X64_SHA256: 5b20c21b2825f69b89eb37baa657f0f0062124517e6e6828e9857c7e9bbd3070 KOVA_REPOSITORY: openclaw/Kova - KOVA_CANONICAL_CONFIG_REF: dfafaff9dcd49b9c76788c6260f1f72dd2ced593 - KOVA_LEGACY_LIST_CONFIG_REF: dfafaff9dcd49b9c76788c6260f1f72dd2ced593 + KOVA_CANONICAL_CONFIG_REF: 1fe2f4081877bb12b7f7ed355349f98b8a0a6882 + KOVA_LEGACY_LIST_CONFIG_REF: 1fe2f4081877bb12b7f7ed355349f98b8a0a6882 + KOVA_TRUSTED_LIVE_REF: 1fe2f4081877bb12b7f7ed355349f98b8a0a6882 PERFORMANCE_MODEL_ID: gpt-5.6-luna # Release matrices cold-build the candidate runtime before measurement. KOVA_SCENARIO_TIMEOUT_MS: ${{ inputs.profile == 'release' && '900000' || '300000' }} @@ -173,7 +174,7 @@ jobs: echo "tested_sha=$resolved_sha" >> "$GITHUB_OUTPUT" echo "kova_ref=$kova_ref" >> "$GITHUB_OUTPUT" echo "kova_config_contract=$kova_config_contract" >> "$GITHUB_OUTPUT" - if [[ "$kova_ref" == "$KOVA_CANONICAL_CONFIG_REF" || "$kova_ref" == "$KOVA_LEGACY_LIST_CONFIG_REF" ]]; then + if [[ "$kova_ref" == "$KOVA_TRUSTED_LIVE_REF" ]]; then echo "kova_ref_trusted_for_live=true" >> "$GITHUB_OUTPUT" else echo "kova_ref_trusted_for_live=false" >> "$GITHUB_OUTPUT" diff --git a/test/scripts/openclaw-performance-workflow.test.ts b/test/scripts/openclaw-performance-workflow.test.ts index ac5d0965741a..8dd9a52f3edc 100644 --- a/test/scripts/openclaw-performance-workflow.test.ts +++ b/test/scripts/openclaw-performance-workflow.test.ts @@ -127,8 +127,9 @@ describe("OpenClaw performance workflow", () => { it("pins the Kova evaluator with release validation contracts", () => { const workflow = readFileSync(WORKFLOW, "utf8"); - const canonicalKovaRef = "dfafaff9dcd49b9c76788c6260f1f72dd2ced593"; - const legacyKovaRef = "dfafaff9dcd49b9c76788c6260f1f72dd2ced593"; + const canonicalKovaRef = "1fe2f4081877bb12b7f7ed355349f98b8a0a6882"; + const legacyKovaRef = "1fe2f4081877bb12b7f7ed355349f98b8a0a6882"; + const trustedLiveKovaRef = "1fe2f4081877bb12b7f7ed355349f98b8a0a6882"; const install = findStep("Install OCM and Kova"); const installRun = install.run ?? ""; const targetCheckout = findStep("Checkout target metadata", "resolve_target"); @@ -136,6 +137,7 @@ describe("OpenClaw performance workflow", () => { expect(workflow).toContain(`KOVA_CANONICAL_CONFIG_REF: ${canonicalKovaRef}`); expect(workflow).toContain(`KOVA_LEGACY_LIST_CONFIG_REF: ${legacyKovaRef}`); + expect(workflow).toContain(`KOVA_TRUSTED_LIVE_REF: ${trustedLiveKovaRef}`); expect(workflow).toContain("kova_config_contract:"); expect(workflow).toContain("Optional fixture-contract override for a custom Kova ref"); expect(readWorkflow().jobs?.resolve_target?.outputs?.kova_ref).toBe( @@ -177,9 +179,7 @@ describe("OpenClaw performance workflow", () => { expect(resolveTarget.run).toContain( 'echo "kova_config_contract=$kova_config_contract" >> "$GITHUB_OUTPUT"', ); - expect(resolveTarget.run).toContain( - 'if [[ "$kova_ref" == "$KOVA_CANONICAL_CONFIG_REF" || "$kova_ref" == "$KOVA_LEGACY_LIST_CONFIG_REF" ]]; then', - ); + expect(resolveTarget.run).toContain('if [[ "$kova_ref" == "$KOVA_TRUSTED_LIVE_REF" ]]; then'); expect(resolveTarget.run).toContain( 'echo "kova_ref_trusted_for_live=true" >> "$GITHUB_OUTPUT"', ); @@ -215,16 +215,57 @@ describe("OpenClaw performance workflow", () => { }); it("keeps live credentials away from custom Kova refs", () => { + const resolveTarget = findStep("Resolve OpenClaw target ref", "resolve_target"); const decideLane = findStep("Decide lane"); const configureLiveAuth = findStep("Configure live OpenAI auth"); const runKova = findStep("Run Kova"); const root = mkdtempSync(join(realpathSync(tmpdir()), "openclaw-kova-live-ref-")); - const output = join(root, "output"); + const trustedRef = "1fe2f4081877bb12b7f7ed355349f98b8a0a6882"; + const compatibleUntrustedRef = "0f9e678e239b45db46d2bd930b7983203580df78"; const decideLaneRun = (decideLane.run ?? "") .replaceAll("${{ github.event_name }}", "workflow_dispatch") .replaceAll("${{ inputs.deep_profile || 'false' }}", "false") .replaceAll("${{ inputs.live_openai_candidate || 'false' }}", "true"); + const runBoundary = (kovaRef: string, name: string) => { + const resolveOutput = join(root, `${name}-resolve-output`); + const laneOutput = join(root, `${name}-lane-output`); + const resolve = spawnSync("bash", ["-c", resolveTarget.run ?? ""], { + encoding: "utf8", + env: { + ...process.env, + GITHUB_OUTPUT: resolveOutput, + GITHUB_REF_NAME: "fix/kova-runtime-major-baseline", + KOVA_CANONICAL_CONFIG_REF: compatibleUntrustedRef, + KOVA_CONFIG_CONTRACT_INPUT: "canonical", + KOVA_LEGACY_LIST_CONFIG_REF: compatibleUntrustedRef, + KOVA_REF_INPUT: kovaRef, + KOVA_TRUSTED_LIVE_REF: trustedRef, + TARGET_CHECKOUT_DIR: process.cwd(), + TARGET_REF_INPUT: "test-head", + }, + }); + expect(resolve.status, resolve.stderr).toBe(0); + const resolved = Object.fromEntries( + readFileSync(resolveOutput, "utf8") + .trim() + .split("\n") + .map((line) => line.split("=", 2)), + ); + const lane = spawnSync("bash", ["-c", decideLaneRun], { + encoding: "utf8", + env: { + ...process.env, + GITHUB_OUTPUT: laneOutput, + GITHUB_STEP_SUMMARY: join(root, `${name}-summary`), + KOVA_REF_TRUSTED_FOR_LIVE: resolved.kova_ref_trusted_for_live, + LANE_ID: "live-openai-candidate", + SECRET_ELIGIBLE: "true", + }, + }); + return { lane, laneOutput, resolved }; + }; + expect(decideLane.run).toContain( 'if [[ "$LANE_ID" == "live-openai-candidate" && "$run_lane" == "true" && "$KOVA_REF_TRUSTED_FOR_LIVE" != "true" ]]; then', ); @@ -245,33 +286,18 @@ describe("OpenClaw performance workflow", () => { ); try { - const rejected = spawnSync("bash", ["-c", decideLaneRun], { - encoding: "utf8", - env: { - ...process.env, - GITHUB_OUTPUT: output, - KOVA_REF_TRUSTED_FOR_LIVE: "false", - LANE_ID: "live-openai-candidate", - SECRET_ELIGIBLE: "true", - }, - }); - expect(rejected.status).toBe(1); - expect(rejected.stdout).toContain( + const rejected = runBoundary(compatibleUntrustedRef, "untrusted"); + expect(rejected.resolved.kova_ref_trusted_for_live).toBe("false"); + expect(rejected.lane.status).toBe(1); + expect(rejected.lane.stdout).toContain( "The live OpenAI lane only executes a reviewed immutable Kova default.", ); + expect(existsSync(rejected.laneOutput)).toBe(false); - const accepted = spawnSync("bash", ["-c", decideLaneRun], { - encoding: "utf8", - env: { - ...process.env, - GITHUB_OUTPUT: output, - KOVA_REF_TRUSTED_FOR_LIVE: "true", - LANE_ID: "live-openai-candidate", - SECRET_ELIGIBLE: "true", - }, - }); - expect(accepted.status).toBe(0); - expect(readFileSync(output, "utf8")).toContain("run=true\n"); + const accepted = runBoundary(trustedRef, "trusted"); + expect(accepted.resolved.kova_ref_trusted_for_live).toBe("true"); + expect(accepted.lane.status).toBe(0); + expect(readFileSync(accepted.laneOutput, "utf8")).toContain("run=true\n"); } finally { rmSync(root, { force: true, recursive: true }); }