mirror of
https://github.com/open-policy-agent/opa.git
synced 2026-08-26 02:04:50 -06:00
Anders Eknert
d0350b326e
Originally meant to be `array.concat_n`, but this name is better as the behavior of this function differs from `array.concat` — namely that `array.flatten` accepts any type of valued in the input array. Only arrays are however flattened, and the rest are appended directly to the flattened output. Note that this function only flattens at the topmost level of the input array — not recursively! A cursory look at a few other languages suggest a single level is the common case. But if others feel we should flstten more, I'm happy to make an update. The C code for a Wasm implementstion here is cowboy coded, and I did not manage to run the tests on my machine due to some `docker` <-> `container` differences. I mostly just imitated the existing code in the array category. I doubt it'll work on the first try, but only CI can judge me. Also: - Remove `opa fmt` step from the Rego CI step, as this is done by Regal anyway a little later in the list of tasks. - Replace some hard-coded `docker` names in the `Makefile` with `$(DOCKER)` - Added name of built-in function missing to the unsupportedBuiltinErr error, as it has happened a few times now that I've used `:=` in a query, and had no clue what built-in it referred to. Fixes #8226 Signed-off-by: Anders Eknert <anders.eknert@apple.com>
526 lines
16 KiB
YAML
526 lines
16 KiB
YAML
name: PR Check
|
|
|
|
on: [pull_request]
|
|
|
|
# When a new revision is pushed to a PR, cancel all in-progress CI runs for that
|
|
# PR. See https://docs.github.com/en/actions/using-jobs/using-concurrency
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
jobs:
|
|
# Check what types of changes this PR contains
|
|
check-changes:
|
|
name: Check what files changed
|
|
runs-on: ubuntu-24.04
|
|
outputs:
|
|
go: ${{ steps.changes.outputs.go }}
|
|
wasm: ${{ steps.changes.outputs.wasm }}
|
|
docs: ${{ steps.changes.outputs.docs }}
|
|
rego: ${{ steps.changes.outputs.rego }}
|
|
steps:
|
|
- name: Check out repository code
|
|
uses: actions/checkout@v6
|
|
|
|
- name: Download OPA
|
|
uses: open-policy-agent/setup-opa@87c881550699b0257d7b7aeb8c33019d70bae4a2 # v2.3.0
|
|
with:
|
|
version: latest
|
|
|
|
- name: Check for file changes
|
|
id: changes
|
|
run: |
|
|
set -e
|
|
|
|
# Default to running all checks
|
|
echo "go=true" >> $GITHUB_OUTPUT
|
|
echo "wasm=true" >> $GITHUB_OUTPUT
|
|
echo "docs=true" >> $GITHUB_OUTPUT
|
|
echo "rego=true" >> $GITHUB_OUTPUT
|
|
|
|
if ! curl -s -o changed_files.json -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
|
|
"https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files"; then
|
|
echo "Error: Failed to fetch changed files from GitHub API"
|
|
echo "Defaulting to running all checks (go=true, wasm=true, docs=true, rego=true)"
|
|
exit 0
|
|
fi
|
|
|
|
if [ ! -s changed_files.json ]; then
|
|
echo "Warning: No changed files found"
|
|
echo "Defaulting to running all checks (go=true, wasm=true, docs=true, rego=true)"
|
|
exit 0
|
|
fi
|
|
|
|
echo "Changed files:"
|
|
jq -r '.[].filename' changed_files.json
|
|
|
|
opa eval \
|
|
--data build/policy/pr-check/pr_check.rego \
|
|
--input changed_files.json \
|
|
--format pretty \
|
|
'data.policy["pr-check"]' > opa_result.json
|
|
|
|
go_result=$(jq -r '.changes.go // false' opa_result.json)
|
|
wasm_result=$(jq -r '.changes.wasm // false' opa_result.json)
|
|
docs_result=$(jq -r '.changes.docs // false' opa_result.json)
|
|
rego_result=$(jq -r '.changes.rego // false' opa_result.json)
|
|
|
|
echo "go=${go_result}" >> $GITHUB_OUTPUT
|
|
echo "wasm=${wasm_result}" >> $GITHUB_OUTPUT
|
|
echo "docs=${docs_result}" >> $GITHUB_OUTPUT
|
|
echo "rego=${rego_result}" >> $GITHUB_OUTPUT
|
|
|
|
echo "Final outputs:"
|
|
echo " go=${go_result}"
|
|
echo " wasm=${wasm_result}"
|
|
echo " docs=${docs_result}"
|
|
echo " rego=${rego_result}"
|
|
|
|
# All jobs essentially re-create the `ci-release-test` make target, but are split
|
|
# up for parallel runners for faster PR feedback and a nicer UX.
|
|
generate:
|
|
name: Generate Code
|
|
runs-on: ubuntu-24.04
|
|
needs: check-changes
|
|
if: ${{ needs.check-changes.outputs.go == 'true' }}
|
|
steps:
|
|
- name: Check out code
|
|
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
|
|
|
- name: Generate
|
|
run: make clean generate
|
|
|
|
- name: Upload generated artifacts
|
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
|
|
with:
|
|
name: generated
|
|
path: |
|
|
internal/compiler/wasm/opa
|
|
capabilities.json
|
|
|
|
go-build:
|
|
name: Go Build (${{ matrix.os }}${{ matrix.arch && format(' {0}', matrix.arch) || '' }}${{ matrix.go_tags }})
|
|
runs-on: ${{ matrix.run }}
|
|
needs: [generate, check-changes]
|
|
if: ${{ needs.check-changes.outputs.go == 'true' }}
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- os: linux
|
|
run: ubuntu-24.04
|
|
targets: ci-go-ci-build-linux ci-go-ci-build-linux-static
|
|
arch: amd64
|
|
- os: linux
|
|
run: ubuntu-24.04
|
|
targets: ci-go-ci-build-linux ci-go-ci-build-linux-static
|
|
arch: arm64
|
|
- os: windows
|
|
run: ubuntu-24.04
|
|
targets: ci-build-windows
|
|
arch: amd64
|
|
- os: darwin
|
|
run: macos-15-intel
|
|
targets: ci-build-darwin
|
|
arch: amd64
|
|
- os: darwin
|
|
run: macos-15
|
|
targets: ci-build-darwin ci-build-darwin-arm64-static
|
|
arch: arm64
|
|
- # NB(sr): We're only building this to see that it still builds.
|
|
# The resulting binary is not used in any other way.
|
|
os: linux
|
|
run: ubuntu-24.04
|
|
targets: ci-build-linux-static
|
|
go_tags: GO_TAGS="-tags=opa_no_oci"
|
|
arch: arm64
|
|
steps:
|
|
- name: Check out code
|
|
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
|
|
|
- id: go_version
|
|
name: Read go version
|
|
run: echo "go_version=$(cat .go-version)" >> $GITHUB_OUTPUT
|
|
|
|
- name: Install Go (${{ steps.go_version.outputs.go_version }})
|
|
uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6.1.0
|
|
with:
|
|
go-version: ${{ steps.go_version.outputs.go_version }}
|
|
if: matrix.os != 'linux'
|
|
- uses: mlugg/setup-zig@fa65c4058643678a4e4a9a60513944a7d8d35440 # v2.1.0
|
|
with:
|
|
version: '0.15.2'
|
|
if: matrix.os == 'windows'
|
|
|
|
- name: Download generated artifacts
|
|
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
|
|
with:
|
|
name: generated
|
|
|
|
- name: Build
|
|
run: make ${{ matrix.go_tags }} ${{ matrix.targets }}
|
|
env:
|
|
GOARCH: ${{ matrix.arch }}
|
|
timeout-minutes: 30
|
|
|
|
- name: Upload binaries - No Go tags
|
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
|
|
if: ${{ matrix.go_tags == '' }}
|
|
with:
|
|
name: binaries-${{ matrix.os }}-${{ matrix.arch }}
|
|
path: _release
|
|
|
|
go-test:
|
|
name: Go Test (${{ matrix.os }})
|
|
runs-on: ${{ matrix.run }}
|
|
needs: [generate, check-changes]
|
|
if: ${{ needs.check-changes.outputs.go == 'true' }}
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- os: linux
|
|
run: ubuntu-24.04
|
|
- os: darwin
|
|
run: macos-15
|
|
steps:
|
|
- name: Check out code
|
|
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
|
|
|
- id: go_version
|
|
name: Read go version
|
|
run: echo "go_version=$(cat .go-version)" >> $GITHUB_OUTPUT
|
|
|
|
- name: Install Go (${{ steps.go_version.outputs.go_version }})
|
|
uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6.1.0
|
|
with:
|
|
go-version: ${{ steps.go_version.outputs.go_version }}
|
|
|
|
- name: Install Node
|
|
uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6.1.0
|
|
|
|
- name: Download generated artifacts
|
|
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
|
|
with:
|
|
name: generated
|
|
|
|
- name: Unit Test Golang
|
|
run: make test-coverage
|
|
timeout-minutes: 30
|
|
|
|
- name: E2E Test Golang
|
|
run: make e2e
|
|
|
|
go-lint:
|
|
name: Go Lint
|
|
runs-on: ubuntu-24.04
|
|
needs: check-changes
|
|
if: ${{ needs.check-changes.outputs.go == 'true' }}
|
|
steps:
|
|
- name: Check out code
|
|
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
|
|
|
- name: Golang Style and Lint Check
|
|
run: make check
|
|
timeout-minutes: 30
|
|
|
|
yaml-lint:
|
|
name: YAML Lint
|
|
runs-on: ubuntu-24.04
|
|
needs: check-changes
|
|
if: ${{ needs.check-changes.outputs.go == 'true' }}
|
|
steps:
|
|
- name: Check out code
|
|
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
|
|
|
- name: YAML Style and Lint Check
|
|
run: make check-yaml-tests
|
|
timeout-minutes: 30
|
|
env:
|
|
YAML_LINT_FORMAT: github
|
|
|
|
wasm:
|
|
name: WASM
|
|
runs-on: ubuntu-24.04
|
|
needs: [generate, check-changes]
|
|
if: ${{ needs.check-changes.outputs.wasm == 'true' }}
|
|
steps:
|
|
- name: Check out code
|
|
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
|
|
|
- name: Download generated artifacts
|
|
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
|
|
with:
|
|
name: generated
|
|
|
|
- name: Build and Test Wasm
|
|
run: make ci-wasm
|
|
timeout-minutes: 15
|
|
|
|
- name: Build and Test Wasm SDK
|
|
run: make ci-go-wasm-sdk-e2e-test
|
|
timeout-minutes: 30
|
|
env:
|
|
DOCKER_RUNNING: 0
|
|
|
|
check-generated:
|
|
name: Check Generated
|
|
runs-on: ubuntu-24.04
|
|
needs: [generate, check-changes]
|
|
if: ${{ needs.check-changes.outputs.go == 'true' }}
|
|
steps:
|
|
- name: Check out code
|
|
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
|
|
|
- name: Download generated artifacts
|
|
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
|
|
with:
|
|
name: generated
|
|
|
|
- name: Check Working Copy
|
|
run: make ci-check-working-copy
|
|
timeout-minutes: 15
|
|
env:
|
|
DOCKER_RUNNING: 0
|
|
|
|
race-detector:
|
|
name: Go Race Detector
|
|
runs-on: ubuntu-24.04
|
|
needs: [generate, check-changes]
|
|
if: ${{ needs.check-changes.outputs.go == 'true' }}
|
|
steps:
|
|
- name: Check out code
|
|
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
|
|
|
- name: Download generated artifacts
|
|
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
|
|
with:
|
|
name: generated
|
|
|
|
- name: Test with Race Detector
|
|
run: make ci-go-race-detector
|
|
env:
|
|
DOCKER_RUNNING: 0
|
|
|
|
smoke-test-docker-images:
|
|
name: docker image smoke test
|
|
runs-on: ubuntu-24.04
|
|
needs: [go-build, check-changes]
|
|
if: ${{ needs.check-changes.outputs.go == 'true' }}
|
|
steps:
|
|
- name: Check out code
|
|
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
|
|
|
- name: Set up QEMU
|
|
uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0
|
|
with:
|
|
platforms: arm64
|
|
|
|
- name: Download release binaries
|
|
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
|
|
with:
|
|
pattern: binaries-*
|
|
merge-multiple: true
|
|
path: _release
|
|
|
|
- name: Test amd64 images
|
|
run: make ci-image-smoke-test
|
|
|
|
- name: Test arm64 images
|
|
run: make ci-image-smoke-test
|
|
env:
|
|
GOARCH: arm64
|
|
|
|
# Note(philipc): We only run the amd64 targets for windows/linux
|
|
smoke-test-binaries:
|
|
runs-on: ${{ matrix.run }}
|
|
needs: [go-build, check-changes]
|
|
if: ${{ needs.check-changes.outputs.go == 'true' }}
|
|
strategy:
|
|
matrix:
|
|
include:
|
|
- os: linux
|
|
run: ubuntu-24.04
|
|
exec: opa_linux_amd64
|
|
arch: amd64
|
|
- os: linux
|
|
run: ubuntu-24.04
|
|
exec: opa_linux_amd64_static
|
|
arch: amd64
|
|
wasm: disabled
|
|
- os: darwin
|
|
run: macos-15-intel
|
|
exec: opa_darwin_amd64
|
|
arch: amd64
|
|
- os: darwin
|
|
run: macos-15
|
|
exec: opa_darwin_arm64_static
|
|
arch: arm64
|
|
wasm: disabled
|
|
- os: windows
|
|
run: windows-latest
|
|
exec: opa_windows_amd64.exe
|
|
arch: amd64
|
|
|
|
steps:
|
|
- name: Check out code
|
|
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
|
|
|
- name: Install Go
|
|
uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6.1.0
|
|
with:
|
|
go-version: stable
|
|
|
|
- name: Download release binaries
|
|
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
|
|
with:
|
|
name: binaries-${{ matrix.os }}-${{ matrix.arch }}
|
|
path: _release
|
|
|
|
- name: Prep tests
|
|
run: go install github.com/rogpeppe/go-internal/cmd/testscript@latest
|
|
|
|
- name: CLI E2E tests
|
|
run: |
|
|
matches=($BINARY_PATH_GLOB) # expand glob
|
|
export OPA="$(pwd)/${matches[0]}"
|
|
chmod +x "$OPA"
|
|
find . -type f -name '*.txtar' -path '*/script/*' -print0 \
|
|
| xargs -0 -I{} testscript -e OPA {}
|
|
shell: bash
|
|
env:
|
|
BINARY_PATH_GLOB: _release/*/${{ matrix.exec }}
|
|
|
|
- name: wasm smoke test
|
|
run: _release/*/${{ matrix.exec }} eval --target wasm 'time.now_ns()'
|
|
shell: bash
|
|
if: matrix.wasm != 'disabled'
|
|
|
|
go-version-build:
|
|
name: Go compat build/test
|
|
needs: [generate, check-changes]
|
|
if: ${{ needs.check-changes.outputs.go == 'true' }}
|
|
runs-on: ${{ matrix.os }}
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
os: [ubuntu-24.04, macos-15]
|
|
version: ["1.24"]
|
|
steps:
|
|
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
|
- name: Download generated artifacts
|
|
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
|
|
with:
|
|
name: generated
|
|
- uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6.1.0
|
|
with:
|
|
go-version: ${{ matrix.version }}
|
|
- run: make build
|
|
env:
|
|
DOCKER_RUNNING: 0
|
|
- run: make go-test
|
|
env:
|
|
DOCKER_RUNNING: 0
|
|
|
|
# Run PR metadata against Rego policies
|
|
rego-check-pr:
|
|
name: Rego PR checks
|
|
runs-on: ubuntu-24.04
|
|
needs: check-changes
|
|
if: ${{ needs.check-changes.outputs.rego == 'true' }}
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
|
|
|
- name: Download OPA
|
|
uses: open-policy-agent/setup-opa@950f159a49aa91f9323f36f1de81c7f6b5de9576 # v2.3.0
|
|
with:
|
|
version: edge
|
|
|
|
- name: Test policies
|
|
run: opa test --schema build/policy/schema --bundle build/policy
|
|
|
|
- name: Run file policy checks on changed files
|
|
run: |
|
|
curl --silent --fail --header 'Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' -o files.json \
|
|
https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files
|
|
|
|
opa eval --bundle build/policy --format values --input files.json --fail-defined 'data.files.deny[message]'
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
- name: Download Regal
|
|
uses: StyraInc/setup-regal@33a142b1189004e0f14bf42b15972c67eecce776 #v1.0.0
|
|
with:
|
|
version: latest
|
|
|
|
- name: Run Regal lint
|
|
# Current configuration ensures anything but build/policy is ignored. While this could point Regal only at that
|
|
# directory, this will serve as a reminder when more Rego policies are added, as they should be linted by default.
|
|
run: regal lint --format github .
|
|
|
|
docs-build:
|
|
name: Build Docs
|
|
runs-on: ubuntu-24.04
|
|
needs: check-changes
|
|
if: ${{ needs.check-changes.outputs.docs == 'true' }}
|
|
steps:
|
|
- name: Check out code
|
|
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
|
|
|
- name: Build docs
|
|
run: make docs-install docs-build
|
|
|
|
# This job is required to complete before merging, and is set as a branch
|
|
# protection rule:
|
|
# https://github.com/open-policy-agent/opa/settings/branch_protection_rules
|
|
pr-check-summary:
|
|
name: PR Check Summary
|
|
runs-on: ubuntu-24.04
|
|
needs: [
|
|
check-changes,
|
|
generate,
|
|
go-build,
|
|
go-test,
|
|
go-lint,
|
|
yaml-lint,
|
|
wasm,
|
|
check-generated,
|
|
race-detector,
|
|
smoke-test-docker-images,
|
|
smoke-test-binaries,
|
|
go-version-build,
|
|
rego-check-pr,
|
|
docs-build,
|
|
]
|
|
if: always()
|
|
steps:
|
|
- name: Check out code
|
|
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
|
- name: Download OPA
|
|
uses: open-policy-agent/setup-opa@950f159a49aa91f9323f36f1de81c7f6b5de9576 # v2.3.0
|
|
with:
|
|
version: edge
|
|
- name: Check job results
|
|
run: |
|
|
# Create the input file with all job results
|
|
echo '${{ toJSON(needs) }}' > input.json
|
|
|
|
# Find failed or cancelled jobs using OPA
|
|
opa eval -d .github/workflows/pull-request.yaml \
|
|
--input=input.json \
|
|
'{job|some _, job in data.jobs["pr-check-summary"].needs} & {job | input[job].result in {"failure", "cancelled"}}' \
|
|
--format=raw > failed_jobs.json
|
|
|
|
# Check for failures and display a nice message
|
|
if [ "$(cat failed_jobs.json)" != "[]" ]; then
|
|
echo "The following required jobs did not complete successfully:"
|
|
jq -r '.[]' failed_jobs.json | sed 's/^/- /'
|
|
exit 1
|
|
fi
|
|
|
|
echo "All jobs completed successfully or were skipped"
|