diff --git a/.github/workflows/benchmarks.yaml b/.github/workflows/benchmarks.yaml index 5943ee5272..a9769ebde8 100644 --- a/.github/workflows/benchmarks.yaml +++ b/.github/workflows/benchmarks.yaml @@ -74,6 +74,87 @@ jobs: publish: true publish_branch: benchmarks + regression-check: + permissions: + contents: read + pull-requests: write + name: Check for regressions + runs-on: ubuntu-24.04 + needs: [check-changes, benchmarks] + if: ${{ needs.check-changes.outputs.go == 'true' }} + steps: + - name: Check out code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - id: go_version + name: Read go version + run: echo "go_version=$(cat .go-version)" >> $GITHUB_OUTPUT + - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + go-version: ${{ steps.go_version.outputs.go_version }} + - name: Fetch benchmark data + run: | + git fetch origin benchmarks + git show origin/benchmarks:benchmarks.json > benchmarks.json + - name: Split latest two runs + run: | + # Extract the two most recent runs into separate files + jq '.[0]' benchmarks.json | jq '[.]' > current.json + jq '.[1]' benchmarks.json | jq '[.]' > base.json + - name: Run regression checks + id: checks + continue-on-error: true + run: | + go run go.bobheadxi.dev/gobenchdata@v1 checks eval base.json current.json \ + --checks.config build/gobenchdata-checks.yml \ + --json report.json + go run go.bobheadxi.dev/gobenchdata@v1 checks report report.json + - name: Comment on PR if regression detected + if: ${{ steps.checks.outcome == 'failure' }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COMMIT_SHA: ${{ github.sha }} + run: | + PR_NUMBER=$(gh pr list --search "${COMMIT_SHA}" --state merged --json number --jq '.[0].number') + + if [ -z "$PR_NUMBER" ] || [ "$PR_NUMBER" = "null" ]; then + echo "Could not find originating PR for commit ${COMMIT_SHA}" + exit 0 + fi + + REPORT=$(jq -r ' + .checks[] + | select(.status != "pass") + | "| \(.name) | \(.status) | \(.diffs | length) benchmarks |" + ' report.json) + + BODY=$(cat < + Full report + + \`\`\`json + $(jq . report.json) + \`\`\` + + + + [Benchmarks Dashboard](https://open-policy-agent.github.io/opa/index.html) + + _This comment was automatically generated by the benchmarks workflow._ + EOF + ) + + gh pr comment "${PR_NUMBER}" --body "${BODY}" + notebook: permissions: contents: write # we'll push to the `benchmarks` branch diff --git a/build/gobenchdata-checks.yml b/build/gobenchdata-checks.yml new file mode 100644 index 0000000000..b067d58318 --- /dev/null +++ b/build/gobenchdata-checks.yml @@ -0,0 +1,8 @@ +checks: + - name: ns-per-op regression + description: Flag if NsPerOp regresses by more than 10% + package: "" + benchmarks: [] + diff: (current.NsPerOp - base.NsPerOp) / base.NsPerOp * 100 + thresholds: + max: 10