benchmarks: use benchlab for per-PR feedback

Hopefully makes stuff a little more robust.

Signed-off-by: Stephan Renatus <stephan.renatus@gmail.com>

.github/workflows/benchmarks.yaml

@@ -80,55 +80,47 @@ jobs:
       pull-requests: write
     name: Check for regressions
     runs-on: ubuntu-24.04
-    needs: [check-changes, benchmarks]
-    if: ${{ needs.check-changes.outputs.go == 'true' }}
+    needs: [check-changes]
+    # if: ${{ needs.check-changes.outputs.go == 'true' }} # TODO(sr): re-enable when this works
     steps:
       - name: Check out code
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
+          fetch-depth: 1
           persist-credentials: false
+      - name: Fetch base commit
+        env:
+          BEFORE_SHA: ${{ github.event.before }}
+        run: git fetch --depth=1 origin "$BEFORE_SHA"
       - id: go_version
         name: Read go version
         run: echo "go_version=$(cat .go-version)" >> $GITHUB_OUTPUT
       - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
         with:
           go-version: ${{ steps.go_version.outputs.go_version }}
-      - uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4.0.1
-        with:
-          install: true
-          cache: true
-          mise_toml: |
-            [tools]
-            babashka = "1.12.218"
-      - name: Fetch benchmark data
+      - name: Install benchlab # this fork has some improvements over rsc.io/benchlab; but we cannot `go install` it directly
         run: |
-          git fetch origin benchmarks
-          git show origin/benchmarks:benchmarks.json > benchmarks.json
-      - name: Split latest two runs
+          git clone https://github.com/FiloSottile/rsc-cmd /tmp/rsc-cmd
+          git -C /tmp/rsc-cmd checkout 9fc40f0f0431e88dc7f088cc6238d0d6ed1d77d0
+          cd /tmp/rsc-cmd/benchlab && go install .
+      - name: Run benchmarks
+        env:
+          BEFORE_SHA: ${{ github.event.before }}
+          AFTER_SHA: ${{ github.event.after }}
         run: |
-          # Extract the two most recent runs into separate files
-          jq '.[0]' benchmarks.json | jq '[.]' > current.json
-          jq '.[1]' benchmarks.json | jq '[.]' > base.json
-      - name: Run regression checks
-        id: checks
-        continue-on-error: true
-        run: |
-          go run go.bobheadxi.dev/gobenchdata@v1 checks eval base.json current.json \
-            --checks.config build/gobenchdata-checks.yml \
-            --json report.json
-          go run go.bobheadxi.dev/gobenchdata@v1 checks report report.json
+          for pkg in ./v1/rego ./v1/topdown ./v1/ast; do
+            benchlab \
+              -commit "$BEFORE_SHA","$AFTER_SHA" \
+              -pkg "$pkg" \
+              -host local:tags=opa_wasm \
+              -reps 3 \
+              -benchtime 300ms
+          done
       - name: Comment on PR with benchmark results
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           COMMIT_SHA: ${{ github.sha }}
         run: |
-          bb build/benchmark-regression-comment.bb report.json > body.md
-
-          if [ ! -s body.md ]; then
-            echo "No regressions or notable improvements; skipping comment."
-            exit 0
-          fi
-
           PR_NUMBER=$(gh pr list --search "${COMMIT_SHA}" --state merged --json number --jq '.[0].number')
 
           if [ -z "$PR_NUMBER" ] || [ "$PR_NUMBER" = "null" ]; then
@@ -136,6 +128,21 @@ jobs:
             exit 0
           fi
 
+          {
+            echo "## Benchmark Comparison (HEAD^ vs HEAD)"
+            echo ""
+            echo '```'
+            # Each benchstat file ends with a compact benchmark × host delta table;
+            # extract from that header line onward so the comment stays readable.
+            for f in .benchlab/benchstat.*.txt; do
+              awk '/^benchmark \\ host/{found=1} found{print}' "$f"
+              echo ""
+            done
+            echo '```'
+            echo ""
+            echo "_This comment was automatically generated by the benchmarks workflow._"
+          } > body.md
+
           gh pr comment "${PR_NUMBER}" --body-file body.md
 
   notebook:

build/benchmark-regression-comment.bb

@@ -1,66 +0,0 @@
-#!/usr/bin/env bb
-
-(require '[cheshire.core :as json]
-         '[clojure.string :as str])
-
-(def base-url "https://open-policy-agent.github.io/opa")
-(def opa-prefix "github.com/open-policy-agent/opa")
-
-(defn benchmark-id [pkg bench-name]
-  (-> (str pkg "_" bench-name)
-      (str/replace #"[^a-zA-Z0-9]" "-")
-      (str/replace #"-" "_")))
-
-(defn benchmark-url [pkg bench-name]
-  (let [short-pkg (str/replace pkg opa-prefix ".")
-        id (benchmark-id short-pkg bench-name)]
-    (str base-url "/benchmarks." id ".html")))
-
-(defn short-pkg [pkg]
-  (str/replace pkg (str opa-prefix "/") ""))
-
-(def improvement-threshold -35)
-
-(let [report (json/parse-string (slurp (first *command-line-args*)) true)
-      commit (System/getenv "COMMIT_SHA")
-      all-diffs (->> (:Checks report)
-                     (mapcat (fn [[_ v]] (:Diffs v))))
-      failed (->> (:Checks report)
-                  (filter (fn [[_ v]] (not= (:Status v) "pass")))
-                  (mapcat (fn [[_ v]] (:Diffs v)))
-                  (filter (fn [d] (not= (:Status d) "pass")))
-                  (sort-by :Value >))
-      improved (->> all-diffs
-                    (filter (fn [d] (<= (:Value d) improvement-threshold)))
-                    (sort-by :Value))]
-  (when (or (seq failed) (seq improved))
-    (when (seq failed)
-      (println "## Benchmark Regression Detected")
-      (println)
-      (printf "Commit `%s` introduced benchmark regressions (threshold: >25%% ns/op increase).\n\n" commit)
-      (println "| Package | Benchmark | Regression |")
-      (println "|---------|-----------|-----------|")
-      (doseq [{:keys [Package Benchmark Value]} failed]
-        (printf "| %s | [%s](%s) | +%d%% |\n"
-                (short-pkg Package)
-                Benchmark
-                (benchmark-url Package Benchmark)
-                (Math/round (double Value))))
-      (println))
-    (when (seq improved)
-      (println "## Notable Improvements :tada:")
-      (println)
-      (printf "Commit `%s` delivered big speedups (>%d%% ns/op decrease). Nice work!\n\n"
-              commit
-              (Math/abs improvement-threshold))
-      (println "| Package | Benchmark | Improvement |")
-      (println "|---------|-----------|-------------|")
-      (doseq [{:keys [Package Benchmark Value]} improved]
-        (printf "| %s | [%s](%s) | %d%% |\n"
-                (short-pkg Package)
-                Benchmark
-                (benchmark-url Package Benchmark)
-                (Math/round (double Value))))
-      (println))
-    (printf "[Benchmarks Dashboard](%s/index.html)\n\n" base-url)
-    (println "_This comment was automatically generated by the benchmarks workflow._")))

build/gobenchdata-checks.yml

@@ -1,8 +0,0 @@
-checks:
-  - name: ns-per-op regression
-    description: Flag if NsPerOp regresses by more than 25%
-    package: ""
-    benchmarks: []
-    diff: (current.NsPerOp - base.NsPerOp) / base.NsPerOp * 100
-    thresholds:
-      max: 25