diff --git a/.github/workflows/openclaw-performance.yml b/.github/workflows/openclaw-performance.yml index e3b85def4865..81aa0897a810 100644 --- a/.github/workflows/openclaw-performance.yml +++ b/.github/workflows/openclaw-performance.yml @@ -93,6 +93,8 @@ jobs: kova_ref: ${{ steps.resolve.outputs.kova_ref }} kova_config_contract: ${{ steps.resolve.outputs.kova_config_contract }} kova_ref_trusted_for_live: ${{ steps.resolve.outputs.kova_ref_trusted_for_live }} + secret_eligible: ${{ steps.candidate_trust.outputs.secret_eligible }} + cache_write_allowed: ${{ steps.candidate_trust.outputs.cache_write_allowed }} steps: - name: Checkout target metadata uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 @@ -177,6 +179,34 @@ jobs: echo "kova_ref_trusted_for_live=false" >> "$GITHUB_OUTPUT" fi + - name: Classify performance candidate trust + id: candidate_trust + env: + CANDIDATE_SHA: ${{ steps.resolve.outputs.tested_sha }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + WORKFLOW_SHA: ${{ github.workflow_sha }} + shell: bash + run: | + set -euo pipefail + + secret_eligible=false + cache_write_allowed=false + if [[ + "$GITHUB_EVENT_NAME" == "schedule" || + "$GITHUB_EVENT_NAME" == "workflow_dispatch" + ]] && [[ + "$GITHUB_REF" == "refs/heads/${DEFAULT_BRANCH}" && + "$CANDIDATE_SHA" == "$WORKFLOW_SHA" + ]]; then + secret_eligible=true + cache_write_allowed=true + fi + + { + echo "secret_eligible=$secret_eligible" + echo "cache_write_allowed=$cache_write_allowed" + } >> "$GITHUB_OUTPUT" + kova: name: ${{ matrix.title }} needs: resolve_target @@ -234,6 +264,7 @@ jobs: MATRIX_DEEP_PROFILE: ${{ matrix.deep_profile }} MATRIX_LIVE: ${{ matrix.live }} KOVA_REF_TRUSTED_FOR_LIVE: ${{ needs.resolve_target.outputs.kova_ref_trusted_for_live }} + SECRET_ELIGIBLE: ${{ needs.resolve_target.outputs.secret_eligible }} steps: - name: Decide lane id: lane @@ -250,6 +281,10 @@ jobs: run_lane=false reason="live_openai_candidate input is false" fi + if [[ "$LANE_ID" == "live-openai-candidate" && "$run_lane" == "true" && "$SECRET_ELIGIBLE" != "true" ]]; then + run_lane=false + reason="candidate is not eligible for live credentials" + fi if [[ "$LANE_ID" == "live-openai-candidate" && "$run_lane" == "true" && "$KOVA_REF_TRUSTED_FOR_LIVE" != "true" ]]; then echo "::error::The live OpenAI lane only executes a reviewed immutable Kova default. Omit kova_ref or update the pinned workflow defaults after review." exit 1 @@ -271,7 +306,7 @@ jobs: if: steps.lane.outputs.run == 'true' uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: - ref: ${{ github.sha }} + ref: ${{ github.workflow_sha }} path: .artifacts/performance-workflow fetch-depth: 1 persist-credentials: false @@ -295,11 +330,35 @@ jobs: echo "Workflow SHA: ${GITHUB_SHA}" } >> "$GITHUB_STEP_SUMMARY" + - name: Stage trusted setup action graph + if: steps.lane.outputs.run == 'true' + shell: bash + run: &stage_trusted_performance_setup | + set -euo pipefail + github_dir="$GITHUB_WORKSPACE/.github" + actions_dir="$github_dir/actions" + trusted_action="$PERFORMANCE_HELPER_DIR/.github/actions/setup-pnpm-store-cache" + + test -f "$trusted_action/action.yml" + test -f "$trusted_action/ensure-node.sh" + if [[ -L "$github_dir" || ( -e "$github_dir" && ! -d "$github_dir" ) ]]; then + rm -rf -- "$github_dir" + fi + mkdir -p "$github_dir" + if [[ -L "$actions_dir" || ( -e "$actions_dir" && ! -d "$actions_dir" ) ]]; then + rm -rf -- "$actions_dir" + fi + mkdir -p "$actions_dir" + rm -rf -- "$actions_dir/setup-pnpm-store-cache" + cp -R -- "$trusted_action" "$actions_dir/setup-pnpm-store-cache" + cmp "$trusted_action/action.yml" "$actions_dir/setup-pnpm-store-cache/action.yml" + cmp "$trusted_action/ensure-node.sh" "$actions_dir/setup-pnpm-store-cache/ensure-node.sh" + - name: Set up Node environment if: steps.lane.outputs.run == 'true' - uses: ./.github/actions/setup-node-env + uses: ./.artifacts/performance-workflow/.github/actions/setup-node-env with: - cache-mode: restore + cache-mode: ${{ needs.resolve_target.outputs.cache_write_allowed == 'true' && 'restore' || 'off' }} install-bun: "false" - name: Prepare systemd user session @@ -448,7 +507,7 @@ jobs: echo "KOVA_LANE_REPEAT=$repeat" >> "$GITHUB_ENV" - name: Configure live OpenAI auth - if: ${{ steps.lane.outputs.run == 'true' && matrix.live == 'true' }} + if: ${{ steps.lane.outputs.run == 'true' && matrix.live == 'true' && needs.resolve_target.outputs.secret_eligible == 'true' }} env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL }} @@ -466,8 +525,8 @@ jobs: id: kova if: steps.lane.outputs.run == 'true' env: - OPENAI_API_KEY: ${{ matrix.live == 'true' && secrets.OPENAI_API_KEY || '' }} - OPENAI_BASE_URL: ${{ matrix.live == 'true' && secrets.OPENAI_BASE_URL || '' }} + OPENAI_API_KEY: ${{ matrix.live == 'true' && needs.resolve_target.outputs.secret_eligible == 'true' && secrets.OPENAI_API_KEY || '' }} + OPENAI_BASE_URL: ${{ matrix.live == 'true' && needs.resolve_target.outputs.secret_eligible == 'true' && secrets.OPENAI_BASE_URL || '' }} shell: bash run: | set -euo pipefail @@ -646,7 +705,7 @@ jobs: - name: Checkout source performance helpers uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: - ref: ${{ github.sha }} + ref: ${{ github.workflow_sha }} path: .artifacts/performance-workflow fetch-depth: 1 persist-credentials: false @@ -667,10 +726,14 @@ jobs: echo "Workflow SHA: ${GITHUB_SHA}" } >> "$GITHUB_STEP_SUMMARY" + - name: Stage trusted source setup action graph + shell: bash + run: *stage_trusted_performance_setup + - name: Set up source performance environment - uses: ./.github/actions/setup-node-env + uses: ./.artifacts/performance-workflow/.github/actions/setup-node-env with: - cache-mode: restore + cache-mode: ${{ needs.resolve_target.outputs.cache_write_allowed == 'true' && 'restore' || 'off' }} install-bun: "false" - name: Fetch previous source performance baseline @@ -925,7 +988,7 @@ jobs: publish: name: Publish ${{ matrix.title }} report needs: [resolve_target, kova, source_performance] - if: ${{ always() && (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.publish_reports == true)) && needs.resolve_target.result == 'success' && needs.kova.result != 'cancelled' && needs.source_performance.result != 'cancelled' }} + if: ${{ always() && needs.resolve_target.outputs.secret_eligible == 'true' && (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.publish_reports == true)) && needs.resolve_target.result == 'success' && needs.kova.result != 'cancelled' && needs.source_performance.result != 'cancelled' }} runs-on: ubuntu-24.04 timeout-minutes: 30 permissions: @@ -966,7 +1029,7 @@ jobs: if: steps.lane.outputs.run == 'true' uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: - ref: ${{ github.sha }} + ref: ${{ github.workflow_sha }} path: .artifacts/performance-publisher sparse-checkout: | scripts/lib/kova-report-publish-files.mjs @@ -1246,7 +1309,7 @@ jobs: - name: Create clawgrit reports app token id: clawgrit_app_token - if: ${{ steps.prepare.outputs.ready == 'true' && steps.prepare.outputs.already_published != 'true' }} + if: ${{ needs.resolve_target.outputs.secret_eligible == 'true' && steps.prepare.outputs.ready == 'true' && steps.prepare.outputs.already_published != 'true' }} continue-on-error: ${{ env.REPORT_PUBLISH_REQUIRED != 'true' }} uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 with: @@ -1257,7 +1320,7 @@ jobs: permission-contents: write - name: Publish to clawgrit reports - if: ${{ steps.prepare.outputs.ready == 'true' && steps.prepare.outputs.already_published != 'true' }} + if: ${{ needs.resolve_target.outputs.secret_eligible == 'true' && steps.prepare.outputs.ready == 'true' && steps.prepare.outputs.already_published != 'true' }} continue-on-error: ${{ env.REPORT_PUBLISH_REQUIRED != 'true' }} env: CLAWGRIT_REPORTS_APP_TOKEN: ${{ steps.clawgrit_app_token.outputs.token }} diff --git a/test/scripts/openclaw-performance-workflow.test.ts b/test/scripts/openclaw-performance-workflow.test.ts index cad75f99ea91..dc66077c4475 100644 --- a/test/scripts/openclaw-performance-workflow.test.ts +++ b/test/scripts/openclaw-performance-workflow.test.ts @@ -74,6 +74,46 @@ function kovaMatrixEntries(): Array> { return readWorkflow().jobs?.kova?.strategy?.matrix?.include ?? []; } +function runCandidateTrustClassification({ + candidateSha, + eventName, + ref, + workflowSha, +}: { + candidateSha: string; + eventName: "schedule" | "workflow_dispatch"; + ref: string; + workflowSha: string; +}) { + const step = findStep("Classify performance candidate trust", "resolve_target"); + const root = tempDirs.make("openclaw-performance-candidate-trust-"); + const output = join(root, "output"); + const result = spawnSync("bash", ["-c", step.run ?? ""], { + encoding: "utf8", + env: { + ...process.env, + CANDIDATE_SHA: candidateSha, + DEFAULT_BRANCH: "main", + GITHUB_EVENT_NAME: eventName, + GITHUB_OUTPUT: output, + GITHUB_REF: ref, + WORKFLOW_SHA: workflowSha, + }, + }); + const outputs = Object.fromEntries( + existsSync(output) + ? readFileSync(output, "utf8") + .trim() + .split("\n") + .map((line) => { + const separator = line.indexOf("="); + return [line.slice(0, separator), line.slice(separator + 1)]; + }) + : [], + ); + return { outputs, result }; +} + describe("OpenClaw performance workflow", () => { it("uses an optional dispatch identifier to name parent-owned runs", () => { const workflow = readFileSync(WORKFLOW, "utf8"); @@ -195,13 +235,13 @@ describe("OpenClaw performance workflow", () => { decideLane.run?.indexOf('echo "run=$run_lane"') ?? -1, ); expect(configureLiveAuth.if).toBe( - "${{ steps.lane.outputs.run == 'true' && matrix.live == 'true' }}", + "${{ steps.lane.outputs.run == 'true' && matrix.live == 'true' && needs.resolve_target.outputs.secret_eligible == 'true' }}", ); expect(runKova.env?.OPENAI_API_KEY).toBe( - "${{ matrix.live == 'true' && secrets.OPENAI_API_KEY || '' }}", + "${{ matrix.live == 'true' && needs.resolve_target.outputs.secret_eligible == 'true' && secrets.OPENAI_API_KEY || '' }}", ); expect(runKova.env?.OPENAI_BASE_URL).toBe( - "${{ matrix.live == 'true' && secrets.OPENAI_BASE_URL || '' }}", + "${{ matrix.live == 'true' && needs.resolve_target.outputs.secret_eligible == 'true' && secrets.OPENAI_BASE_URL || '' }}", ); try { @@ -212,6 +252,7 @@ describe("OpenClaw performance workflow", () => { GITHUB_OUTPUT: output, KOVA_REF_TRUSTED_FOR_LIVE: "false", LANE_ID: "live-openai-candidate", + SECRET_ELIGIBLE: "true", }, }); expect(rejected.status).toBe(1); @@ -226,6 +267,7 @@ describe("OpenClaw performance workflow", () => { GITHUB_OUTPUT: output, KOVA_REF_TRUSTED_FOR_LIVE: "true", LANE_ID: "live-openai-candidate", + SECRET_ELIGIBLE: "true", }, }); expect(accepted.status).toBe(0); @@ -235,6 +277,157 @@ describe("OpenClaw performance workflow", () => { } }); + it("keeps arbitrary performance candidates secretless and cacheless", () => { + const workflow = readWorkflow(); + const trust = findStep("Classify performance candidate trust", "resolve_target"); + const decideLane = findStep("Decide lane"); + const kovaHarness = findStep("Checkout performance workflow helpers"); + const kovaStage = findStep("Stage trusted setup action graph"); + const kovaSetup = findStep("Set up Node environment"); + const sourceHarness = findStep("Checkout source performance helpers", "source_performance"); + const sourceStage = findStep("Stage trusted source setup action graph", "source_performance"); + const sourceSetup = findStep("Set up source performance environment", "source_performance"); + const publisherHarness = findStep("Checkout performance publisher helper", "publish"); + + expect(workflow.jobs?.resolve_target?.outputs).toMatchObject({ + secret_eligible: "${{ steps.candidate_trust.outputs.secret_eligible }}", + cache_write_allowed: "${{ steps.candidate_trust.outputs.cache_write_allowed }}", + }); + expect(trust.env).toMatchObject({ + CANDIDATE_SHA: "${{ steps.resolve.outputs.tested_sha }}", + DEFAULT_BRANCH: "${{ github.event.repository.default_branch }}", + WORKFLOW_SHA: "${{ github.workflow_sha }}", + }); + expect(trust.run).toContain("secret_eligible=false"); + expect(trust.run).toContain("cache_write_allowed=false"); + expect(trust.run).toContain('"$GITHUB_REF" == "refs/heads/${DEFAULT_BRANCH}"'); + expect(trust.run).toContain('"$CANDIDATE_SHA" == "$WORKFLOW_SHA"'); + expect(trust.run).toContain("secret_eligible=true"); + expect(trust.run).toContain("cache_write_allowed=true"); + + for (const harness of [kovaHarness, sourceHarness, publisherHarness]) { + expect(harness.with?.ref).toBe("${{ github.workflow_sha }}"); + expect(harness.with?.["persist-credentials"]).toBe(false); + } + for (const setup of [kovaSetup, sourceSetup]) { + expect(setup.uses).toBe("./.artifacts/performance-workflow/.github/actions/setup-node-env"); + expect(setup.with?.["cache-mode"]).toBe( + "${{ needs.resolve_target.outputs.cache_write_allowed == 'true' && 'restore' || 'off' }}", + ); + } + expect(kovaStage.run).toBe(sourceStage.run); + for (const stage of [kovaStage, sourceStage]) { + expect(stage.run).toContain( + 'trusted_action="$PERFORMANCE_HELPER_DIR/.github/actions/setup-pnpm-store-cache"', + ); + expect(stage.run).toContain('rm -rf -- "$actions_dir/setup-pnpm-store-cache"'); + expect(stage.run).toContain( + 'cp -R -- "$trusted_action" "$actions_dir/setup-pnpm-store-cache"', + ); + expect(stage.run).toContain( + 'cmp "$trusted_action/action.yml" "$actions_dir/setup-pnpm-store-cache/action.yml"', + ); + expect(stage.run).toContain( + 'cmp "$trusted_action/ensure-node.sh" "$actions_dir/setup-pnpm-store-cache/ensure-node.sh"', + ); + } + const kovaSteps = workflow.jobs?.kova?.steps ?? []; + const sourceSteps = workflow.jobs?.source_performance?.steps ?? []; + expect(kovaSteps.findIndex((step) => step.name === kovaStage.name)).toBeLessThan( + kovaSteps.findIndex((step) => step.name === kovaSetup.name), + ); + expect(sourceSteps.findIndex((step) => step.name === sourceStage.name)).toBeLessThan( + sourceSteps.findIndex((step) => step.name === sourceSetup.name), + ); + expect(decideLane.run).toContain( + 'if [[ "$LANE_ID" == "live-openai-candidate" && "$run_lane" == "true" && "$SECRET_ELIGIBLE" != "true" ]]; then', + ); + expect(decideLane.run).toContain('reason="candidate is not eligible for live credentials"'); + + const trustedSha = "a".repeat(40); + for (const eventName of ["schedule", "workflow_dispatch"] as const) { + const trusted = runCandidateTrustClassification({ + candidateSha: trustedSha, + eventName, + ref: "refs/heads/main", + workflowSha: trustedSha, + }); + expect(trusted.result.status, trusted.result.stderr).toBe(0); + expect(trusted.outputs).toEqual({ + secret_eligible: "true", + cache_write_allowed: "true", + }); + } + + for (const candidate of [ + { + candidateSha: "b".repeat(40), + eventName: "workflow_dispatch" as const, + ref: "refs/heads/main", + workflowSha: trustedSha, + }, + { + candidateSha: trustedSha, + eventName: "workflow_dispatch" as const, + ref: "refs/heads/release/2026.8.1", + workflowSha: trustedSha, + }, + ]) { + const untrusted = runCandidateTrustClassification(candidate); + expect(untrusted.result.status, untrusted.result.stderr).toBe(0); + expect(untrusted.outputs).toEqual({ + secret_eligible: "false", + cache_write_allowed: "false", + }); + } + }); + + it("replaces candidate-owned nested setup actions with the trusted workflow copy", () => { + const stages = [ + findStep("Stage trusted setup action graph"), + findStep("Stage trusted source setup action graph", "source_performance"), + ]; + + for (const stage of stages) { + const root = tempDirs.make("openclaw-performance-action-graph-"); + const workspace = join(root, "candidate"); + const helper = join(root, "workflow"); + const candidateAction = join(workspace, ".github/actions/setup-pnpm-store-cache/action.yml"); + const candidateEnsureNode = join( + workspace, + ".github/actions/setup-pnpm-store-cache/ensure-node.sh", + ); + const trustedAction = join(helper, ".github/actions/setup-pnpm-store-cache/action.yml"); + const trustedEnsureNode = join( + helper, + ".github/actions/setup-pnpm-store-cache/ensure-node.sh", + ); + mkdirSync(join(workspace, ".github/actions/setup-pnpm-store-cache"), { + recursive: true, + }); + mkdirSync(join(helper, ".github/actions/setup-pnpm-store-cache"), { + recursive: true, + }); + writeFileSync(candidateAction, "candidate action\n"); + writeFileSync(candidateEnsureNode, "candidate script\n"); + writeFileSync(trustedAction, "trusted action\n"); + writeFileSync(trustedEnsureNode, "trusted script\n"); + + const result = spawnSync("bash", ["-c", stage.run ?? ""], { + encoding: "utf8", + env: { + ...process.env, + GITHUB_WORKSPACE: workspace, + PERFORMANCE_HELPER_DIR: helper, + }, + }); + + expect(result.status, result.stderr).toBe(0); + expect(readFileSync(candidateAction, "utf8")).toBe("trusted action\n"); + expect(readFileSync(candidateEnsureNode, "utf8")).toBe("trusted script\n"); + } + }); + it("pins the OCM release archive and checksum", () => { const workflow = readFileSync(WORKFLOW, "utf8"); const installRun = findStep("Install OCM and Kova").run ?? ""; @@ -425,7 +618,7 @@ describe("OpenClaw performance workflow", () => { expect(publisher?.needs).toEqual(["resolve_target", "kova", "source_performance"]); expect(publisher?.if).toBe( - "${{ always() && (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.publish_reports == true)) && needs.resolve_target.result == 'success' && needs.kova.result != 'cancelled' && needs.source_performance.result != 'cancelled' }}", + "${{ always() && needs.resolve_target.outputs.secret_eligible == 'true' && (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.publish_reports == true)) && needs.resolve_target.result == 'success' && needs.kova.result != 'cancelled' && needs.source_performance.result != 'cancelled' }}", ); expect(publisher?.["runs-on"]).toBe("ubuntu-24.04"); expect(publisher?.permissions?.actions).toBe("read"); @@ -488,7 +681,7 @@ describe("OpenClaw performance workflow", () => { expect(appToken.id).toBe("clawgrit_app_token"); expect(appToken.if).toBe( - "${{ steps.prepare.outputs.ready == 'true' && steps.prepare.outputs.already_published != 'true' }}", + "${{ needs.resolve_target.outputs.secret_eligible == 'true' && steps.prepare.outputs.ready == 'true' && steps.prepare.outputs.already_published != 'true' }}", ); expect(appToken.uses).toBe( "actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1", @@ -506,7 +699,7 @@ describe("OpenClaw performance workflow", () => { expect(workflowText.split(appTokenOutput)).toHaveLength(2); expect(workflowText.split("${{ secrets.CLAWSWEEPER_APP_PRIVATE_KEY }}")).toHaveLength(2); expect(publish.if).toBe( - "${{ steps.prepare.outputs.ready == 'true' && steps.prepare.outputs.already_published != 'true' }}", + "${{ needs.resolve_target.outputs.secret_eligible == 'true' && steps.prepare.outputs.ready == 'true' && steps.prepare.outputs.already_published != 'true' }}", ); expect(workflowText).not.toContain("CLAWGRIT_REPORTS_TOKEN"); expect(workflowText).not.toContain("secrets.GH_APP_PRIVATE_KEY"); @@ -622,7 +815,7 @@ describe("OpenClaw performance workflow", () => { "scripts/lib/kova-report-selector.mjs", ); expect(helper.with).toMatchObject({ - ref: "${{ github.sha }}", + ref: "${{ github.workflow_sha }}", path: ".artifacts/performance-publisher", "sparse-checkout": "scripts/lib/kova-report-publish-files.mjs\nscripts/lib/kova-report-selector.mjs\n", @@ -1110,16 +1303,17 @@ esac const runKova = findStep("Run Kova"); expect(configureAuth.if).toContain("matrix.live == 'true'"); + expect(configureAuth.if).toContain("needs.resolve_target.outputs.secret_eligible == 'true'"); expect(configureAuth.env?.OPENAI_API_KEY).toBe("${{ secrets.OPENAI_API_KEY }}"); expect(configureAuth.run).toContain('if [[ -z "${OPENAI_API_KEY:-}" ]]; then'); expect(configureAuth.run).toContain("cannot run without live evidence"); expect(configureAuth.run).toContain("exit 1"); expect(configureAuth.run).not.toContain("will be skipped"); expect(runKova.env?.OPENAI_API_KEY).toBe( - "${{ matrix.live == 'true' && secrets.OPENAI_API_KEY || '' }}", + "${{ matrix.live == 'true' && needs.resolve_target.outputs.secret_eligible == 'true' && secrets.OPENAI_API_KEY || '' }}", ); expect(runKova.env?.OPENAI_BASE_URL).toBe( - "${{ matrix.live == 'true' && secrets.OPENAI_BASE_URL || '' }}", + "${{ matrix.live == 'true' && needs.resolve_target.outputs.secret_eligible == 'true' && secrets.OPENAI_BASE_URL || '' }}", ); expect(runKova.run).not.toContain('echo "skipped=true" >> "$GITHUB_OUTPUT"'); });